Add configuration common.indexSliceSize (#16438)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

configs/milvus.yaml

@@ -274,7 +274,7 @@ common:
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
   # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building.
   simdType: auto
+  indexSliceSize: 4 # MB
 
   storage:
     vector: minio
-

internal/core/src/config/ConfigKnowhere.cpp

@@ -62,4 +62,9 @@ KnowhereSetSimdType(const char* value) {
     }
 }
 
+void
+KnowhereSetIndexSliceSize(const int64_t size) {
+    knowhere::KnowhereConfig::SetIndexFileSliceSize(size);
+}
+
 }  // namespace milvus::config

internal/core/src/config/ConfigKnowhere.h

@@ -25,4 +25,7 @@ KnowhereInitImpl();
 std::string
 KnowhereSetSimdType(const char*);
 
+void
+KnowhereSetIndexSliceSize(const int64_t size);
+
 }  // namespace milvus::config

internal/core/src/indexbuilder/init_c.cpp

@@ -27,3 +27,8 @@ IndexBuilderSetSimdType(const char* value) {
     ret[real_type.length()] = 0;
     return ret;
 }
+
+void
+IndexBuilderSetIndexSliceSize(const int64_t value) {
+    milvus::config::KnowhereSetIndexSliceSize(value);
+}

internal/core/src/indexbuilder/init_c.h

@@ -22,6 +22,9 @@ IndexBuilderInit();
 char*
 IndexBuilderSetSimdType(const char*);
 
+void
+IndexBuilderSetIndexSliceSize(const int64_t);
+
 #ifdef __cplusplus
 };
 #endif

internal/core/src/segcore/segcore_init_c.cpp

@@ -50,4 +50,10 @@ SegcoreSetSimdType(const char* value) {
     return ret;
 }
 
+extern "C" void
+SegcoreSetIndexSliceSize(const int64_t value) {
+    milvus::config::KnowhereSetIndexSliceSize(value);
+    LOG_SEGCORE_DEBUG_ << "set config index slice size: " << value;
+}
+
 }  // namespace milvus::segcore

internal/core/src/segcore/segcore_init_c.h

@@ -25,6 +25,9 @@ SegcoreSetChunkRows(const int64_t);
 char*
 SegcoreSetSimdType(const char*);
 
+void
+SegcoreSetIndexSliceSize(const int64_t);
+
 #ifdef __cplusplus
 }
 #endif

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -11,8 +11,8 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 #-------------------------------------------------------------------------------
 
-set( KNOWHERE_VERSION v1.1.2 )
-set( KNOWHERE_SOURCE_MD5 "7c6043bfaa073c6580a1c5b1e904c995" )
+set( KNOWHERE_VERSION v1.1.3 )
+set( KNOWHERE_SOURCE_MD5 "7497e91053608b354de7ef7f6c6ee54b" )
 
 if ( DEFINED ENV{MILVUS_KNOWHERE_URL} )
     set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" )

internal/indexnode/indexnode.go

@@ -153,6 +153,10 @@ func (i *IndexNode) initKnowhere() {
 	Params.CommonCfg.SimdType = C.GoString(cRealSimdType)
 	C.free(unsafe.Pointer(cRealSimdType))
 	C.free(unsafe.Pointer(cSimdType))
+
+	// override segcore index slice size
+	cIndexSliceSize := C.int64_t(Params.CommonCfg.IndexSliceSize)
+	C.IndexBuilderSetIndexSliceSize(cIndexSliceSize)
 }
 
 func (i *IndexNode) initSession() error {

internal/querynode/query_node.go

@@ -185,6 +185,10 @@ func (node *QueryNode) InitSegcore() {
 	Params.CommonCfg.SimdType = C.GoString(cRealSimdType)
 	C.free(unsafe.Pointer(cRealSimdType))
 	C.free(unsafe.Pointer(cSimdType))
+
+	// override segcore index slice size
+	cIndexSliceSize := C.int64_t(Params.CommonCfg.IndexSliceSize)
+	C.SegcoreSetIndexSliceSize(cIndexSliceSize)
 }
 
 func (node *QueryNode) initServiceDiscovery() error {

internal/util/paramtable/component_param.go

@@ -28,6 +28,9 @@ import (
 const (
 	// DefaultRetentionDuration defines the default duration for retention which is 5 days in seconds.
 	DefaultRetentionDuration = 3600 * 24 * 5
+
+	// DefaultIndexSliceSize defines the default slice size of index file when serializing.
+	DefaultIndexSliceSize = 4
 )
 
 // ComponentParam is used to quickly and easily access all components' configurations.
@@ -115,7 +118,8 @@ type commonConfig struct {
 	DefaultIndexName     string
 	RetentionDuration    int64
 
-	SimdType string
+	SimdType       string
+	IndexSliceSize int64
 }
 
 func (p *commonConfig) init(base *BaseTable) {
@@ -149,6 +153,7 @@ func (p *commonConfig) init(base *BaseTable) {
 	p.initRetentionDuration()
 
 	p.initSimdType()
+	p.initIndexSliceSize()
 }
 
 func (p *commonConfig) initClusterPrefix() {
@@ -325,6 +330,10 @@ func (p *commonConfig) initSimdType() {
 	p.SimdType = p.Base.LoadWithDefault2(keys, "auto")
 }
 
+func (p *commonConfig) initIndexSliceSize() {
+	p.IndexSliceSize = p.Base.ParseInt64WithDefault("common.indexSliceSize", DefaultIndexSliceSize)
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // --- rootcoord ---
 type rootCoordConfig struct {

internal/util/paramtable/component_param_test.go

@@ -40,10 +40,14 @@ func TestComponentParam(t *testing.T) {
 		t.Logf("default index name = %s", Params.DefaultIndexName)
 
 		assert.Equal(t, Params.RetentionDuration, int64(DefaultRetentionDuration))
+		t.Logf("default retention duration = %d", Params.RetentionDuration)
 
 		assert.NotEqual(t, Params.SimdType, "")
 		t.Logf("knowhere simd type = %s", Params.SimdType)
 
+		assert.Equal(t, Params.IndexSliceSize, int64(DefaultIndexSliceSize))
+		t.Logf("knowhere index slice size = %d", Params.IndexSliceSize)
+
 		// -- proxy --
 		assert.Equal(t, Params.ProxySubName, "by-dev-proxy")
 		t.Logf("ProxySubName: %s", Params.ProxySubName)