milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: use global pool but not dedicated pool for every index (#37852) 

issue: #37851

- make a global thread pool at tantivy temporally.
- set 1 but not 4 threads for inverted text index.

Signed-off-by: chyezh <chyezh@outlook.com>
pull/37817/head^2
Zhen Ye 2024-11-20 20:44:32 +08:00 committed by GitHub
parent 714421997c
commit f3a36f8a29
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock generated vendored
View File

@ -781,8 +781,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "ownedbytes"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"stable_deref_trait",
]
@ -1107,8 +1106,7 @@ dependencies = [
[[package]]
name = "tantivy"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6083cd777fa94271b8ce0fe4533772cb8110c3044bab048d20f70108329a1f2"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"aho-corasick",
"arc-swap",
@ -1177,8 +1175,7 @@ dependencies = [
[[package]]
name = "tantivy-bitpacker"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"bitpacking",
]
@ -1186,8 +1183,7 @@ dependencies = [
[[package]]
name = "tantivy-columnar"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"fastdivide",
"fnv",
@ -1202,8 +1198,7 @@ dependencies = [
[[package]]
name = "tantivy-common"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"async-trait",
"byteorder",
@ -1226,8 +1221,7 @@ dependencies = [
[[package]]
name = "tantivy-query-grammar"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"nom",
]
@ -1235,8 +1229,7 @@ dependencies = [
[[package]]
name = "tantivy-sstable"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"tantivy-common",
"tantivy-fst",
@ -1246,8 +1239,7 @@ dependencies = [
[[package]]
name = "tantivy-stacker"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"murmurhash32",
"tantivy-common",
@ -1256,8 +1248,7 @@ dependencies = [
[[package]]
name = "tantivy-tokenizer-api"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66"
source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb"
dependencies = [
"serde",
]

2
internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml vendored
View File

@ -6,7 +6,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tantivy = "=0.21.1"
tantivy = { git = "https://github.com/milvus-io/tantivy", tag = "0.21.1-fix" } # we have make a private fix for milvus, should be removed in future after milvus fixing the bug.
futures = "0.3.21"
libc = "0.2"
scopeguard = "1.2"

3
internal/core/thirdparty/tantivy/tantivy-wrapper.h vendored
View File

@ -15,7 +15,8 @@ using Map = std::map<std::string, std::string>;
static constexpr const char* DEFAULT_TOKENIZER_NAME = "milvus_tokenizer";
static const char* DEFAULT_analyzer_params = "{}";
static constexpr uintptr_t DEFAULT_NUM_THREADS = 4;
static constexpr uintptr_t DEFAULT_NUM_THREADS =
1; // Every field with index writer will generate a thread, make huge thread amount, wait for refactoring.
static constexpr uintptr_t DEFAULT_OVERALL_MEMORY_BUDGET_IN_BYTES =
DEFAULT_NUM_THREADS * 15 * 1024 * 1024;