mirror of https://github.com/milvus-io/milvus.git
fix: [2.5]Replace the position of EnabledJSONKeyStats (#40108)
Replace the position of EnabledJSONKeyStats issue: https://github.com/milvus-io/milvus/issues/36995 pr: https://github.com/milvus-io/milvus/pull/38039 --------- Signed-off-by: Xianhui.Lin <xianhui.lin@zilliz.com>pull/40028/head
parent
2677f0b834
commit
c1de61ff7c
|
@ -506,7 +506,6 @@ indexCoord:
|
||||||
nodeID: 0
|
nodeID: 0
|
||||||
segment:
|
segment:
|
||||||
minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed
|
minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed
|
||||||
enabledJsonKeyStats: true # Indicates whether to enable JSON key stats
|
|
||||||
|
|
||||||
indexNode:
|
indexNode:
|
||||||
scheduler:
|
scheduler:
|
||||||
|
@ -894,6 +893,7 @@ common:
|
||||||
localRPCEnabled: false # enable local rpc for internal communication when mix or standalone mode.
|
localRPCEnabled: false # enable local rpc for internal communication when mix or standalone mode.
|
||||||
sync:
|
sync:
|
||||||
taskPoolReleaseTimeoutSeconds: 60 # The maximum time to wait for the task to finish and release resources in the pool
|
taskPoolReleaseTimeoutSeconds: 60 # The maximum time to wait for the task to finish and release resources in the pool
|
||||||
|
enabledJsonKeyStats: false # Indicates whether to enable JSON key stats
|
||||||
|
|
||||||
# QuotaConfig, configurations of Milvus quota and limits.
|
# QuotaConfig, configurations of Milvus quota and limits.
|
||||||
# By default, we enable:
|
# By default, we enable:
|
||||||
|
|
|
@ -200,7 +200,7 @@ func (jm *statsJobManager) triggerJsonKeyIndexStatsTask() {
|
||||||
needTriggerFieldIDs := make([]UniqueID, 0)
|
needTriggerFieldIDs := make([]UniqueID, 0)
|
||||||
for _, field := range collection.Schema.GetFields() {
|
for _, field := range collection.Schema.GetFields() {
|
||||||
h := typeutil.CreateFieldSchemaHelper(field)
|
h := typeutil.CreateFieldSchemaHelper(field)
|
||||||
if h.EnableJSONKeyIndex() && Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool() {
|
if h.EnableJSONKeyIndex() && Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() {
|
||||||
needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID())
|
needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -123,5 +123,5 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() {
|
||||||
|
|
||||||
jm.loopWg.Wait()
|
jm.loopWg.Wait()
|
||||||
|
|
||||||
s.Equal(3, jm.scheduler.pendingTasks.TaskCount())
|
s.Equal(2, jm.scheduler.pendingTasks.TaskCount())
|
||||||
}
|
}
|
||||||
|
|
|
@ -205,7 +205,7 @@ func (i *IndexNode) initSegcore() {
|
||||||
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
|
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
|
||||||
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
|
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
|
||||||
|
|
||||||
cJSONIndexEnabled := C.bool(Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool())
|
cJSONIndexEnabled := C.bool(Params.CommonCfg.EnabledJSONKeyStats.GetAsBool())
|
||||||
C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled)
|
C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,7 @@ import (
|
||||||
"github.com/milvus-io/milvus/pkg/log"
|
"github.com/milvus-io/milvus/pkg/log"
|
||||||
"github.com/milvus-io/milvus/pkg/proto/datapb"
|
"github.com/milvus-io/milvus/pkg/proto/datapb"
|
||||||
"github.com/milvus-io/milvus/pkg/proto/querypb"
|
"github.com/milvus-io/milvus/pkg/proto/querypb"
|
||||||
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
||||||
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -142,7 +143,7 @@ func (c *StatsChecker) checkSegment(segment *meta.Segment, resp *milvuspb.Descri
|
||||||
var result []int64
|
var result []int64
|
||||||
for _, field := range resp.GetSchema().GetFields() {
|
for _, field := range resp.GetSchema().GetFields() {
|
||||||
h := typeutil.CreateFieldSchemaHelper(field)
|
h := typeutil.CreateFieldSchemaHelper(field)
|
||||||
if h.EnableJSONKeyIndex() {
|
if h.EnableJSONKeyIndex() && paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool() {
|
||||||
exists := false
|
exists := false
|
||||||
for i := 0; i < len(segment.JSONIndexField); i++ {
|
for i := 0; i < len(segment.JSONIndexField); i++ {
|
||||||
if segment.JSONIndexField[i] == field.FieldID {
|
if segment.JSONIndexField[i] == field.FieldID {
|
||||||
|
|
|
@ -94,7 +94,8 @@ func (suite *StatsCheckerSuite) TearDownTest() {
|
||||||
func (suite *StatsCheckerSuite) TestLoadJsonIndex() {
|
func (suite *StatsCheckerSuite) TestLoadJsonIndex() {
|
||||||
checker := suite.checker
|
checker := suite.checker
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
|
||||||
|
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
|
||||||
// meta
|
// meta
|
||||||
coll := utils.CreateTestCollection(1, 1)
|
coll := utils.CreateTestCollection(1, 1)
|
||||||
coll.FieldIndexID = map[int64]int64{101: 1000}
|
coll.FieldIndexID = map[int64]int64{101: 1000}
|
||||||
|
@ -155,7 +156,8 @@ func (suite *StatsCheckerSuite) TestLoadJsonIndex() {
|
||||||
func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() {
|
func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() {
|
||||||
checker := suite.checker
|
checker := suite.checker
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
|
||||||
|
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
|
||||||
// meta
|
// meta
|
||||||
coll := utils.CreateTestCollection(1, 1)
|
coll := utils.CreateTestCollection(1, 1)
|
||||||
coll.FieldIndexID = map[int64]int64{101: 1000}
|
coll.FieldIndexID = map[int64]int64{101: 1000}
|
||||||
|
@ -198,7 +200,8 @@ func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() {
|
||||||
func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() {
|
func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() {
|
||||||
checker := suite.checker
|
checker := suite.checker
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
|
||||||
|
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
|
||||||
// meta
|
// meta
|
||||||
coll := utils.CreateTestCollection(1, 1)
|
coll := utils.CreateTestCollection(1, 1)
|
||||||
coll.FieldIndexID = map[int64]int64{101: 1000}
|
coll.FieldIndexID = map[int64]int64{101: 1000}
|
||||||
|
@ -231,7 +234,8 @@ func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() {
|
||||||
func (suite *StatsCheckerSuite) TestCreateNewJsonIndex() {
|
func (suite *StatsCheckerSuite) TestCreateNewJsonIndex() {
|
||||||
checker := suite.checker
|
checker := suite.checker
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true")
|
||||||
|
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key)
|
||||||
// meta
|
// meta
|
||||||
coll := utils.CreateTestCollection(1, 1)
|
coll := utils.CreateTestCollection(1, 1)
|
||||||
coll.FieldIndexID = map[int64]int64{101: 1000}
|
coll.FieldIndexID = map[int64]int64{101: 1000}
|
||||||
|
|
|
@ -239,6 +239,9 @@ func (node *QueryNode) InitSegcore() error {
|
||||||
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
|
cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64())
|
||||||
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
|
C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval)
|
||||||
|
|
||||||
|
cJSONIndexEnabled := C.bool(paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool())
|
||||||
|
C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled)
|
||||||
|
|
||||||
cGpuMemoryPoolInitSize := C.uint32_t(paramtable.Get().GpuConfig.InitSize.GetAsUint32())
|
cGpuMemoryPoolInitSize := C.uint32_t(paramtable.Get().GpuConfig.InitSize.GetAsUint32())
|
||||||
cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
|
cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
|
||||||
C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)
|
C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)
|
||||||
|
|
|
@ -296,6 +296,8 @@ type commonConfig struct {
|
||||||
LocalRPCEnabled ParamItem `refreshable:"false"`
|
LocalRPCEnabled ParamItem `refreshable:"false"`
|
||||||
|
|
||||||
SyncTaskPoolReleaseTimeoutSeconds ParamItem `refreshable:"true"`
|
SyncTaskPoolReleaseTimeoutSeconds ParamItem `refreshable:"true"`
|
||||||
|
|
||||||
|
EnabledJSONKeyStats ParamItem `refreshable:"true"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *commonConfig) init(base *BaseTable) {
|
func (p *commonConfig) init(base *BaseTable) {
|
||||||
|
@ -998,6 +1000,15 @@ This helps Milvus-CDC synchronize incremental data`,
|
||||||
Export: true,
|
Export: true,
|
||||||
}
|
}
|
||||||
p.SyncTaskPoolReleaseTimeoutSeconds.Init(base.mgr)
|
p.SyncTaskPoolReleaseTimeoutSeconds.Init(base.mgr)
|
||||||
|
|
||||||
|
p.EnabledJSONKeyStats = ParamItem{
|
||||||
|
Key: "common.enabledJsonKeyStats",
|
||||||
|
Version: "2.5.5",
|
||||||
|
DefaultValue: "false",
|
||||||
|
Doc: "Indicates whether to enable JSON key stats",
|
||||||
|
Export: true,
|
||||||
|
}
|
||||||
|
p.EnabledJSONKeyStats.Init(base.mgr)
|
||||||
}
|
}
|
||||||
|
|
||||||
type gpuConfig struct {
|
type gpuConfig struct {
|
||||||
|
@ -3488,8 +3499,6 @@ type dataCoordConfig struct {
|
||||||
MinSegmentNumRowsToEnableIndex ParamItem `refreshable:"true"`
|
MinSegmentNumRowsToEnableIndex ParamItem `refreshable:"true"`
|
||||||
BrokerTimeout ParamItem `refreshable:"false"`
|
BrokerTimeout ParamItem `refreshable:"false"`
|
||||||
|
|
||||||
EnabledJSONKeyStats ParamItem `refreshable:"true"`
|
|
||||||
|
|
||||||
// auto balance channel on datanode
|
// auto balance channel on datanode
|
||||||
AutoBalance ParamItem `refreshable:"true"`
|
AutoBalance ParamItem `refreshable:"true"`
|
||||||
CheckAutoBalanceConfigInterval ParamItem `refreshable:"false"`
|
CheckAutoBalanceConfigInterval ParamItem `refreshable:"false"`
|
||||||
|
@ -4153,15 +4162,6 @@ During compaction, the size of segment # of rows is able to exceed segment max #
|
||||||
}
|
}
|
||||||
p.MinSegmentNumRowsToEnableIndex.Init(base.mgr)
|
p.MinSegmentNumRowsToEnableIndex.Init(base.mgr)
|
||||||
|
|
||||||
p.EnabledJSONKeyStats = ParamItem{
|
|
||||||
Key: "indexCoord.enabledJsonKeyStats",
|
|
||||||
Version: "2.0.0",
|
|
||||||
DefaultValue: "true",
|
|
||||||
Doc: "Indicates whether to enable JSON key stats",
|
|
||||||
Export: true,
|
|
||||||
}
|
|
||||||
p.EnabledJSONKeyStats.Init(base.mgr)
|
|
||||||
|
|
||||||
p.BindIndexNodeMode = ParamItem{
|
p.BindIndexNodeMode = ParamItem{
|
||||||
Key: "indexCoord.bindIndexNodeMode.enable",
|
Key: "indexCoord.bindIndexNodeMode.enable",
|
||||||
Version: "2.0.0",
|
Version: "2.0.0",
|
||||||
|
|
Loading…
Reference in New Issue