fix: [2.5]Replace the position of EnabledJSONKeyStats (#40108)

Replace the position of EnabledJSONKeyStats
issue: https://github.com/milvus-io/milvus/issues/36995
pr: https://github.com/milvus-io/milvus/pull/38039

---------

Signed-off-by: Xianhui.Lin <xianhui.lin@zilliz.com>
pull/40028/head
Xianhui Lin 2025-02-22 14:35:54 +08:00 committed by GitHub
parent 2677f0b834
commit c1de61ff7c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 28 additions and 20 deletions

View File

@ -506,7 +506,6 @@ indexCoord:
nodeID: 0
segment:
minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed
enabledJsonKeyStats: true # Indicates whether to enable JSON key stats
indexNode:
scheduler:
@ -894,6 +893,7 @@ common:
localRPCEnabled: false # enable local rpc for internal communication when mix or standalone mode.
sync:
taskPoolReleaseTimeoutSeconds: 60 # The maximum time to wait for the task to finish and release resources in the pool
enabledJsonKeyStats: false # Indicates whether to enable JSON key stats
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:

View File

@ -200,7 +200,7 @@ func (jm *statsJobManager) triggerJsonKeyIndexStatsTask() {
needTriggerFieldIDs := make([]UniqueID, 0)
for _, field := range collection.Schema.GetFields() {
h := typeutil.CreateFieldSchemaHelper(field)
if h.EnableJSONKeyIndex() && Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool() {
if h.EnableJSONKeyIndex() && Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() {
needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID())
}
}

View File

@ -123,5 +123,5 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() {
jm.loopWg.Wait()
s.Equal(3, jm.scheduler.pendingTasks.TaskCount())
s.Equal(2, jm.scheduler.pendingTasks.TaskCount())
}

View File

@ -205,7 +205,7 @@ func (i *IndexNode) initSegcore() {
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
cJSONIndexEnabled := C.bool(Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool())
cJSONIndexEnabled := C.bool(Params.CommonCfg.EnabledJSONKeyStats.GetAsBool())
C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled)
}

View File

@ -32,6 +32,7 @@ import (
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/proto/datapb"
"github.com/milvus-io/milvus/pkg/proto/querypb"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -142,7 +143,7 @@ func (c *StatsChecker) checkSegment(segment *meta.Segment, resp *milvuspb.Descri
var result []int64
for _, field := range resp.GetSchema().GetFields() {
h := typeutil.CreateFieldSchemaHelper(field)
if h.EnableJSONKeyIndex() {
if h.EnableJSONKeyIndex() && paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool() {
exists := false
for i := 0; i < len(segment.JSONIndexField); i++ {
if segment.JSONIndexField[i] == field.FieldID {

View File

@ -94,7 +94,8 @@ func (suite *StatsCheckerSuite) TearDownTest() {
func (suite *StatsCheckerSuite) TestLoadJsonIndex() {
checker := suite.checker
ctx := context.Background()
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
// meta
coll := utils.CreateTestCollection(1, 1)
coll.FieldIndexID = map[int64]int64{101: 1000}
@ -155,7 +156,8 @@ func (suite *StatsCheckerSuite) TestLoadJsonIndex() {
func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() {
checker := suite.checker
ctx := context.Background()
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
// meta
coll := utils.CreateTestCollection(1, 1)
coll.FieldIndexID = map[int64]int64{101: 1000}
@ -198,7 +200,8 @@ func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() {
func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() {
checker := suite.checker
ctx := context.Background()
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
// meta
coll := utils.CreateTestCollection(1, 1)
coll.FieldIndexID = map[int64]int64{101: 1000}
@ -231,7 +234,8 @@ func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() {
func (suite *StatsCheckerSuite) TestCreateNewJsonIndex() {
checker := suite.checker
ctx := context.Background()
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
// meta
coll := utils.CreateTestCollection(1, 1)
coll.FieldIndexID = map[int64]int64{101: 1000}

View File

@ -239,6 +239,9 @@ func (node *QueryNode) InitSegcore() error {
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
cJSONIndexEnabled := C.bool(paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool())
C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled)
cGpuMemoryPoolInitSize := C.uint32_t(paramtable.Get().GpuConfig.InitSize.GetAsUint32())
cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)

View File

@ -296,6 +296,8 @@ type commonConfig struct {
LocalRPCEnabled ParamItem `refreshable:"false"`
SyncTaskPoolReleaseTimeoutSeconds ParamItem `refreshable:"true"`
EnabledJSONKeyStats ParamItem `refreshable:"true"`
}
func (p *commonConfig) init(base *BaseTable) {
@ -998,6 +1000,15 @@ This helps Milvus-CDC synchronize incremental data`,
Export: true,
}
p.SyncTaskPoolReleaseTimeoutSeconds.Init(base.mgr)
p.EnabledJSONKeyStats = ParamItem{
Key: "common.enabledJsonKeyStats",
Version: "2.5.5",
DefaultValue: "false",
Doc: "Indicates whether to enable JSON key stats",
Export: true,
}
p.EnabledJSONKeyStats.Init(base.mgr)
}
type gpuConfig struct {
@ -3488,8 +3499,6 @@ type dataCoordConfig struct {
MinSegmentNumRowsToEnableIndex ParamItem `refreshable:"true"`
BrokerTimeout ParamItem `refreshable:"false"`
EnabledJSONKeyStats ParamItem `refreshable:"true"`
// auto balance channel on datanode
AutoBalance ParamItem `refreshable:"true"`
CheckAutoBalanceConfigInterval ParamItem `refreshable:"false"`
@ -4153,15 +4162,6 @@ During compaction, the size of segment # of rows is able to exceed segment max #
}
p.MinSegmentNumRowsToEnableIndex.Init(base.mgr)
p.EnabledJSONKeyStats = ParamItem{
Key: "indexCoord.enabledJsonKeyStats",
Version: "2.0.0",
DefaultValue: "true",
Doc: "Indicates whether to enable JSON key stats",
Export: true,
}
p.EnabledJSONKeyStats.Init(base.mgr)
p.BindIndexNodeMode = ParamItem{
Key: "indexCoord.bindIndexNodeMode.enable",
Version: "2.0.0",