milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix get vector it timeout and improve some string const usage (#24141) 

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/24103/head
congqixia 2023-05-16 17:41:22 +08:00 committed by GitHub
parent ccd685013a
commit 73a181d226
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 334 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
internal/datacoord/compaction_trigger_test.go
View File

@ -241,7 +241,7 @@ func Test_compactionTrigger_force(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -261,7 +261,7 @@ func Test_compactionTrigger_force(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -281,7 +281,7 @@ func Test_compactionTrigger_force(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -326,7 +326,7 @@ func Test_compactionTrigger_force(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128error",
},
},
@ -343,7 +343,7 @@ func Test_compactionTrigger_force(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -365,7 +365,7 @@ func Test_compactionTrigger_force(t *testing.T) {
TypeParams: nil,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
},
@ -385,7 +385,7 @@ func Test_compactionTrigger_force(t *testing.T) {
TypeParams: nil,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "DISKANN",
},
},
@ -689,7 +689,7 @@ func Test_compactionTrigger_force_maxSegmentLimit(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -881,7 +881,7 @@ func Test_compactionTrigger_noplan(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -1034,7 +1034,7 @@ func Test_compactionTrigger_PrioritizedCandi(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -1433,7 +1433,7 @@ func Test_compactionTrigger_noplan_random_size(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},

5
internal/datacoord/index_builder_test.go
View File

@ -35,6 +35,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -64,13 +65,13 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
CreateTime: 1,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "metrics_type",
Key: common.MetricTypeKey,
Value: "L2",
},
},

37
internal/datacoord/index_meta_test.go
View File

@ -34,6 +34,7 @@ import (
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/pkg/common"
)
func TestMeta_CanCreateIndex(t *testing.T) {
@ -45,13 +46,13 @@ func TestMeta_CanCreateIndex(t *testing.T) {
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
}
@ -118,7 +119,7 @@ func TestMeta_CanCreateIndex(t *testing.T) {
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
req.TypeParams = []*commonpb.KeyValuePair{{Key: "dim", Value: "64"}}
req.TypeParams = []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "64"}}
tmpIndexID, err = m.CanCreateIndex(req)
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
@ -129,7 +130,7 @@ func TestMeta_CanCreateIndex(t *testing.T) {
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
req.IndexParams = []*commonpb.KeyValuePair{{Key: "index_type", Value: "HNSW"}}
req.IndexParams = []*commonpb.KeyValuePair{{Key: common.IndexTypeKey, Value: "HNSW"}}
tmpIndexID, err = m.CanCreateIndex(req)
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
@ -166,13 +167,13 @@ func TestMeta_HasSameReq(t *testing.T) {
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
}
@ -249,13 +250,13 @@ func TestMeta_CreateIndex(t *testing.T) {
CreateTime: 12,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
},
@ -376,13 +377,13 @@ func TestMeta_GetIndexIDByName(t *testing.T) {
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
}
@ -434,13 +435,13 @@ func TestMeta_GetSegmentIndexState(t *testing.T) {
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
}
@ -555,13 +556,13 @@ func TestMeta_GetSegmentIndexStateOnField(t *testing.T) {
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "FLAT",
},
}
@ -839,13 +840,13 @@ func TestMeta_GetTypeParams(t *testing.T) {
CreateTime: 0,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
},
@ -884,13 +885,13 @@ func TestMeta_GetIndexParams(t *testing.T) {
CreateTime: 0,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
},

37
internal/datacoord/index_service_test.go
View File

@ -35,6 +35,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/common"
)
func TestServerId(t *testing.T) {
@ -50,13 +51,13 @@ func TestServer_CreateIndex(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -123,7 +124,7 @@ func TestServer_CreateIndex(t *testing.T) {
s.meta.indexes = map[UniqueID]map[UniqueID]*model.Index{}
req.IndexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "DISKANN",
},
}
@ -138,7 +139,7 @@ func TestServer_CreateIndex(t *testing.T) {
s.meta.catalog = &datacoord.Catalog{MetaKv: &saveFailKV{}}
req.IndexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -158,13 +159,13 @@ func TestServer_GetIndexState(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -349,13 +350,13 @@ func TestServer_GetSegmentIndexState(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -486,13 +487,13 @@ func TestServer_GetIndexBuildProgress(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -677,13 +678,13 @@ func TestServer_DescribeIndex(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -959,13 +960,13 @@ func TestServer_GetIndexStatistics(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -1239,13 +1240,13 @@ func TestServer_DropIndex(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}
@ -1435,13 +1436,13 @@ func TestServer_GetIndexInfos(t *testing.T) {
indexName = "default_idx"
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}

2
internal/datacoord/mock_test.go
View File

@ -203,7 +203,7 @@ func newTestSchema() *schemapb.CollectionSchema {
Description: "schema for test used",
AutoID: false,
Fields: []*schemapb.FieldSchema{
{FieldID: 1, Name: "field1", IsPrimaryKey: false, Description: "field no.1", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "max_length", Value: "100"}}},
{FieldID: 1, Name: "field1", IsPrimaryKey: false, Description: "field no.1", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: common.MaxLengthKey, Value: "100"}}},
{FieldID: 2, Name: "field2", IsPrimaryKey: false, Description: "field no.2", DataType: schemapb.DataType_FloatVector},
},
}

5
internal/datacoord/segment_allocation_policy_test.go
View File

@ -21,6 +21,7 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/stretchr/testify/assert"
@ -54,7 +55,7 @@ func TestUpperLimitCalBySchema(t *testing.T) {
{
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "bad_dim"},
{Key: common.DimKey, Value: "bad_dim"},
},
},
},
@ -74,7 +75,7 @@ func TestUpperLimitCalBySchema(t *testing.T) {
{
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "128"},
{Key: common.DimKey, Value: "128"},
},
},
},

2
internal/datacoord/util.go
View File

@ -177,7 +177,7 @@ func getCollectionAutoCompactionEnabled(properties map[string]string) (bool, err
func getIndexType(indexParams []*commonpb.KeyValuePair) string {
for _, param := range indexParams {
if param.Key == "index_type" {
if param.Key == common.IndexTypeKey {
return param.Value
}
}

3
internal/datanode/buffer_test.go
View File

@ -34,6 +34,7 @@ import (
"github.com/milvus-io/milvus/internal/datanode/allocator"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -44,7 +45,7 @@ func genTestCollectionSchema(dim int64, vectorType schemapb.DataType) *schemapb.
DataType: vectorType,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: fmt.Sprintf("%d", dim),
},
},

5
internal/datanode/compactor_test.go
View File

@ -38,6 +38,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -419,7 +420,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
_, _, _, err = ct.merge(context.Background(), allPaths, 2, 0, &etcdpb.CollectionMeta{
Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "64"},
{Key: common.DimKey, Value: "64"},
}},
}},
}, dm)
@ -456,7 +457,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
_, _, _, err = ct.merge(context.Background(), allPaths, 2, 0, &etcdpb.CollectionMeta{
Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{
{DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "dim"},
{Key: common.DimKey, Value: "bad_dim"},
}},
}}}, dm)
assert.Error(t, err)

6
internal/datanode/mock_test.go
View File

@ -402,7 +402,7 @@ func (mf *MetaFactory) GetFieldSchema() []*schemapb.FieldSchema {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "2",
},
},
@ -420,7 +420,7 @@ func (mf *MetaFactory) GetFieldSchema() []*schemapb.FieldSchema {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "32",
},
},
@ -494,7 +494,7 @@ func (mf *MetaFactory) GetFieldSchema() []*schemapb.FieldSchema {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "100",
},
},

17
internal/indexnode/indexnode_service_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/metautil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/stretchr/testify/assert"
@ -70,17 +71,17 @@ func TestIndexNodeSimple(t *testing.T) {
vecDim int64 = 8
typeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: fmt.Sprintf("%d", vecDim),
},
}
indexParams = []*commonpb.KeyValuePair{
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "L2",
},
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{
@ -199,15 +200,15 @@ func TestIndexNodeComplex(t *testing.T) {
idxID0 int64 = 40000
typesParamsLists = [][]*commonpb.KeyValuePair{
{{
Key: "dim",
Key: common.DimKey,
Value: fmt.Sprintf("%d", 8),
}},
{{
Key: "dim",
Key: common.DimKey,
Value: fmt.Sprintf("%d", 16),
}},
{{
Key: "dim",
Key: common.DimKey,
Value: fmt.Sprintf("%d", 32),
}},
}
@ -219,11 +220,11 @@ func TestIndexNodeComplex(t *testing.T) {
Value: "128",
},
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "L2",
},
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
}

22
internal/indexnode/indexnode_test.go
View File

@ -123,21 +123,21 @@ import (
// DataPaths: paths,
// TypeParams: []*commonpb.KeyValuePair{
// {
// Key: "dim",
// Key: common.DimKey,
// Value: "8",
// },
// },
// IndexParams: []*commonpb.KeyValuePair{
// {
// Key: "index_type",
// Key: common.IndexTypeKey,
// Value: "IVF_SQ8",
// },
// {
// Key: "params",
// Key: common.IndexParamsKey,
// Value: "{\"nlist\": 128}",
// },
// {
// Key: "metric_type",
// Key: common.MetricTypeKey,
// Value: "L2",
// },
// },
@ -238,17 +238,17 @@ import (
// DataPaths: paths,
// TypeParams: []*commonpb.KeyValuePair{
// {
// Key: "dim",
// Key: common.DimKey,
// Value: "8",
// },
// },
// IndexParams: []*commonpb.KeyValuePair{
// {
// Key: "index_type",
// Key: common.IndexTypeKey,
// Value: "BIN_FLAT",
// },
// {
// Key: "metric_type",
// Key: common.MetricTypeKey,
// Value: "JACCARD",
// },
// },
@ -351,21 +351,21 @@ import (
// DataPaths: paths,
// TypeParams: []*commonpb.KeyValuePair{
// {
// Key: "dim",
// Key: common.DimKey,
// Value: "8",
// },
// },
// IndexParams: []*commonpb.KeyValuePair{
// {
// Key: "index_type",
// Key: common.IndexTypeKey,
// Value: "IVF_SQ8",
// },
// {
// Key: "params",
// Key: common.IndexParamsKey,
// Value: "{\"nlist\": 128}",
// },
// {
// Key: "metric_type",
// Key: common.MetricTypeKey,
// Value: "L2",
// },
// },

9
internal/indexnode/task.go
View File

@ -34,6 +34,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/funcutil"
@ -166,7 +167,7 @@ func (it *indexBuildTask) Prepare(ctx context.Context) error {
it.newIndexParams = indexParams
it.statistic.IndexParams = it.req.GetIndexParams()
// ugly codes to get dimension
if dimStr, ok := typeParams["dim"]; ok {
if dimStr, ok := typeParams[common.DimKey]; ok {
var err error
it.statistic.Dim, err = strconv.ParseInt(dimStr, 10, 64)
if err != nil {
@ -241,7 +242,7 @@ func (it *indexBuildTask) LoadData(ctx context.Context) error {
func (it *indexBuildTask) BuildIndex(ctx context.Context) error {
// support build diskann index
indexType := it.newIndexParams["index_type"]
indexType := it.newIndexParams[common.IndexTypeKey]
if indexType == indexparamcheck.IndexDISKANN {
return it.BuildDiskAnnIndex(ctx)
}
@ -314,7 +315,7 @@ func (it *indexBuildTask) BuildDiskAnnIndex(ctx context.Context) error {
// check index node support disk index
if !Params.IndexNodeCfg.EnableDisk.GetAsBool() {
log.Ctx(ctx).Error("IndexNode don't support build disk index",
zap.String("index type", it.newIndexParams["index_type"]),
zap.String("index type", it.newIndexParams[common.IndexTypeKey]),
zap.Bool("enable disk", Params.IndexNodeCfg.EnableDisk.GetAsBool()))
return errors.New("index node don't support build disk index")
}
@ -414,7 +415,7 @@ func (it *indexBuildTask) BuildDiskAnnIndex(ctx context.Context) error {
func (it *indexBuildTask) SaveIndexFiles(ctx context.Context) error {
// support build diskann index
indexType := it.newIndexParams["index_type"]
indexType := it.newIndexParams[common.IndexTypeKey]
if indexType == indexparamcheck.IndexDISKANN {
return it.SaveDiskAnnIndexFiles(ctx)
}

2
internal/metastore/kv/rootcoord/kv_catalog_test.go
View File

@ -38,7 +38,7 @@ var (
IndexName: indexName,
IndexID: 1,
FieldID: 1,
IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: "STL_SORT"}},
IndexParams: []*commonpb.KeyValuePair{{Key: common.IndexTypeKey, Value: "STL_SORT"}},
IsDeleted: true,
}
)

3
internal/proxy/get_max_len.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -16,7 +17,7 @@ func GetMaxLength(field *schemapb.FieldSchema) (int64, error) {
return 0, merr.WrapErrParameterInvalid(schemapb.DataType_VarChar, field.GetDataType(), msg)
}
h := typeutil.NewKvPairs(append(field.GetIndexParams(), field.GetTypeParams()...))
maxLengthStr, err := h.Get("max_length")
maxLengthStr, err := h.Get(common.MaxLengthKey)
if err != nil {
msg := "max length not found"
return 0, merr.WrapErrParameterInvalid("max length key in type parameters", "not found", msg)

5
internal/proxy/get_max_len_test.go
View File

@ -4,6 +4,7 @@ import (
"testing"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/stretchr/testify/assert"
@ -32,7 +33,7 @@ func TestGetMaxLength(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "not_int_aha",
},
},
@ -46,7 +47,7 @@ func TestGetMaxLength(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "100",
},
},

8
internal/proxy/proxy_test.go
View File

@ -552,7 +552,7 @@ func TestProxy(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -635,7 +635,7 @@ func TestProxy(t *testing.T) {
IndexName: indexName,
ExtraParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
{
@ -643,7 +643,7 @@ func TestProxy(t *testing.T) {
Value: distance.L2,
},
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{
@ -3326,7 +3326,7 @@ func TestProxy(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},

2
internal/proxy/task.go
View File

@ -47,7 +47,7 @@ const (
AnnsFieldKey = "anns_field"
TopKKey = "topk"
NQKey = "nq"
MetricTypeKey = "metric_type"
MetricTypeKey = common.MetricTypeKey
SearchParamsKey = "params"
RoundDecimalKey = "round_decimal"
OffsetKey = "offset"

2
internal/proxy/task_index.go
View File

@ -169,7 +169,7 @@ func (cit *createIndexTask) parseIndexParams() error {
for k, v := range indexParamsMap {
//Currently, it is required that type_params and index_params do not have same keys.
if k == DimKey || k == maxVarCharLengthKey {
if k == DimKey || k == common.MaxLengthKey {
delete(indexParamsMap, k)
continue
}

19
internal/proxy/task_index_test.go
View File

@ -32,6 +32,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
@ -284,7 +285,7 @@ func Test_parseIndexParams(t *testing.T) {
FieldName: "",
ExtraParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
{
@ -292,7 +293,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: "params",
Key: common.IndexParamsKey,
Value: "{\"M\": 48, \"efConstruction\": 64}",
},
{
@ -335,7 +336,7 @@ func Test_parseIndexParams(t *testing.T) {
assert.ElementsMatch(t,
[]*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
{
@ -369,7 +370,7 @@ func Test_parseIndexParams(t *testing.T) {
FieldName: "",
ExtraParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{
@ -377,7 +378,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "L2",
},
{
Key: "params",
Key: common.IndexParamsKey,
Value: "{\"nlist\": 100}",
},
{
@ -415,9 +416,9 @@ func Test_parseIndexParams(t *testing.T) {
t.Run("parse index params 2", func(t *testing.T) {
Params.Save(Params.AutoIndexConfig.Enable.Key, "true")
indexParams := map[string]any{
"index_type": "HNSW",
"M": 10,
"efConstruction": 100,
common.IndexTypeKey: "HNSW",
"M": 10,
"efConstruction": 100,
}
indexParamsStr, err := json.Marshal(indexParams)
assert.NoError(t, err)
@ -428,7 +429,7 @@ func Test_parseIndexParams(t *testing.T) {
assert.ElementsMatch(t,
[]*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "HNSW",
},
{

60
internal/proxy/task_test.go
View File

@ -90,7 +90,7 @@ func constructCollectionSchema(
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -119,7 +119,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_BinaryVector {
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(testVecDim),
},
}
@ -127,7 +127,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
if dataType == schemapb.DataType_VarChar {
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: strconv.Itoa(testMaxVarCharLength),
},
}
@ -211,7 +211,7 @@ func constructCollectionSchemaWithAllType(
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -226,7 +226,7 @@ func constructCollectionSchemaWithAllType(
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -678,7 +678,7 @@ func TestCreateCollectionTask(t *testing.T) {
schema.Fields[idx].DataType == schemapb.DataType_BinaryVector {
schema.Fields[idx].TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "not int",
},
}
@ -696,7 +696,7 @@ func TestCreateCollectionTask(t *testing.T) {
schema.Fields[idx].DataType == schemapb.DataType_BinaryVector {
schema.Fields[idx].TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(Params.ProxyCfg.MaxDimension.GetAsInt() + 1),
},
}
@ -712,7 +712,7 @@ func TestCreateCollectionTask(t *testing.T) {
schema.Fields[1].DataType = schemapb.DataType_BinaryVector
schema.Fields[1].TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(Params.ProxyCfg.MaxDimension.GetAsInt() + 1),
},
}
@ -731,7 +731,7 @@ func TestCreateCollectionTask(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(128),
},
},
@ -2023,7 +2023,7 @@ func Test_createIndexTask_getIndexedField(t *testing.T) {
TypeParams: nil,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -2104,12 +2104,12 @@ func Test_fillDimension(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
}
assert.Error(t, fillDimension(f, map[string]string{"dim": "8"}))
assert.Error(t, fillDimension(f, map[string]string{common.DimKey: "8"}))
})
t.Run("normal", func(t *testing.T) {
@ -2117,14 +2117,14 @@ func Test_fillDimension(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
}
m := map[string]string{}
assert.NoError(t, fillDimension(f, m))
assert.Equal(t, "128", m["dim"])
assert.Equal(t, "128", m[common.DimKey])
})
}
@ -2134,15 +2134,15 @@ func Test_checkTrain(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
}
m := map[string]string{
"index_type": "IVF_FLAT",
"nlist": "1024",
"metric_type": "L2",
common.IndexTypeKey: "IVF_FLAT",
"nlist": "1024",
common.MetricTypeKey: "L2",
}
assert.NoError(t, checkTrain(f, m))
})
@ -2152,7 +2152,7 @@ func Test_checkTrain(t *testing.T) {
DataType: schemapb.DataType_Int64,
}
m := map[string]string{
"index_type": "scalar",
common.IndexTypeKey: "scalar",
}
assert.NoError(t, checkTrain(f, m))
})
@ -2162,16 +2162,16 @@ func Test_checkTrain(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
}
m := map[string]string{
"index_type": "IVF_FLAT",
"nlist": "1024",
"metric_type": "L2",
"dim": "8",
common.IndexTypeKey: "IVF_FLAT",
"nlist": "1024",
common.MetricTypeKey: "L2",
common.DimKey: "8",
}
assert.Error(t, checkTrain(f, m))
})
@ -2181,14 +2181,14 @@ func Test_checkTrain(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
}
m := map[string]string{
"index_type": "IVF_FLAT",
"metric_type": "L2",
common.IndexTypeKey: "IVF_FLAT",
common.MetricTypeKey: "L2",
}
assert.Error(t, checkTrain(f, m))
})
@ -2236,7 +2236,7 @@ func Test_createIndexTask_PreExecute(t *testing.T) {
TypeParams: nil,
IndexParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -2248,7 +2248,7 @@ func Test_createIndexTask_PreExecute(t *testing.T) {
globalMetaCache = cache
cit.req.ExtraParams = []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{
@ -2256,7 +2256,7 @@ func Test_createIndexTask_PreExecute(t *testing.T) {
Value: "1024",
},
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "L2",
},
}

14
internal/proxy/util.go
View File

@ -33,6 +33,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util"
@ -50,9 +51,6 @@ const (
// enableMultipleVectorFields indicates whether to enable multiple vector fields.
enableMultipleVectorFields = false
// maximum length of variable-length strings
maxVarCharLengthKey = "max_length"
defaultMaxVarCharLength = 65535
// DefaultIndexType name of default index type for scalar field
@ -216,7 +214,7 @@ func validateDimension(field *schemapb.FieldSchema) error {
exist := false
var dim int64
for _, param := range field.TypeParams {
if param.Key == "dim" {
if param.Key == common.DimKey {
exist = true
tmp, err := strconv.ParseInt(param.Value, 10, 64)
if err != nil {
@ -242,7 +240,7 @@ func validateDimension(field *schemapb.FieldSchema) error {
func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema) error {
exist := false
for _, param := range field.TypeParams {
if param.Key != maxVarCharLengthKey {
if param.Key != common.MaxLengthKey {
return fmt.Errorf("type param key(max_length) should be specified for varChar field, not %s", param.Key)
}
@ -268,7 +266,7 @@ func validateVectorFieldMetricType(field *schemapb.FieldSchema) error {
return nil
}
for _, params := range field.IndexParams {
if params.Key == "metric_type" {
if params.Key == common.MetricTypeKey {
return nil
}
}
@ -440,7 +438,7 @@ func validateSchema(coll *schemapb.CollectionSchema) error {
if err2 != nil {
return err2
}
dimStr, ok := typeKv["dim"]
dimStr, ok := typeKv[common.DimKey]
if !ok {
return fmt.Errorf("dim not found in type_params for vector field %s(%d)", field.Name, field.FieldID)
}
@ -449,7 +447,7 @@ func validateSchema(coll *schemapb.CollectionSchema) error {
return fmt.Errorf("invalid dim; %s", dimStr)
}
metricTypeStr, ok := indexKv["metric_type"]
metricTypeStr, ok := indexKv[common.MetricTypeKey]
if ok {
err4 := validateMetricType(field.DataType, metricTypeStr)
if err4 != nil {

39
internal/proxy/util_test.go
View File

@ -25,6 +25,7 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/stretchr/testify/assert"
@ -155,7 +156,7 @@ func TestValidateDimension(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "1",
},
},
@ -163,7 +164,7 @@ func TestValidateDimension(t *testing.T) {
assert.Nil(t, validateDimension(fieldSchema))
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: Params.ProxyCfg.MaxDimension.GetValue(),
},
}
@ -172,14 +173,14 @@ func TestValidateDimension(t *testing.T) {
// invalid dim
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "-1",
},
}
assert.NotNil(t, validateDimension(fieldSchema))
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(int(Params.ProxyCfg.MaxDimension.GetAsInt32() + 1)),
},
}
@ -188,21 +189,21 @@ func TestValidateDimension(t *testing.T) {
fieldSchema.DataType = schemapb.DataType_BinaryVector
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
}
assert.Nil(t, validateDimension(fieldSchema))
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(Params.ProxyCfg.MaxDimension.GetAsInt()),
},
}
assert.Nil(t, validateDimension(fieldSchema))
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "9",
},
}
@ -229,7 +230,7 @@ func TestValidateVectorFieldMetricType(t *testing.T) {
}
assert.NotNil(t, validateVectorFieldMetricType(field1))
field1.IndexParams = append(field1.IndexParams, &commonpb.KeyValuePair{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "",
})
assert.Nil(t, validateVectorFieldMetricType(field1))
@ -266,7 +267,7 @@ func TestValidatePrimaryKey(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "100",
},
},
@ -454,21 +455,21 @@ func TestValidateSchema(t *testing.T) {
tp3Good := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
}
tp3Bad1 := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "asdfa",
},
}
tp3Bad2 := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "-1",
},
}
@ -482,43 +483,43 @@ func TestValidateSchema(t *testing.T) {
tp3Bad4 := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
{
Key: "dim",
Key: common.DimKey,
Value: "64",
},
}
ip3Good := []*commonpb.KeyValuePair{
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "IP",
},
}
ip3Bad1 := []*commonpb.KeyValuePair{
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "JACCARD",
},
}
ip3Bad2 := []*commonpb.KeyValuePair{
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "xxxxxx",
},
}
ip3Bad3 := []*commonpb.KeyValuePair{
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "L2",
},
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "IP",
},
}

43
internal/proxy/validate_util_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -73,7 +74,7 @@ func Test_validateUtil_checkVarCharFieldData(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "2",
},
},
@ -102,7 +103,7 @@ func Test_validateUtil_checkVarCharFieldData(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "4",
},
},
@ -131,7 +132,7 @@ func Test_validateUtil_checkVarCharFieldData(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "2",
},
},
@ -279,7 +280,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -320,7 +321,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -406,7 +407,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -445,7 +446,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -488,7 +489,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "8",
},
},
@ -556,7 +557,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -567,7 +568,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -578,7 +579,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "8",
},
},
@ -638,7 +639,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "8",
},
},
@ -702,7 +703,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "1",
},
},
@ -713,7 +714,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -724,7 +725,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "8",
},
},
@ -788,7 +789,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "1",
},
},
@ -799,7 +800,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -810,7 +811,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "2",
},
},
@ -914,7 +915,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -925,7 +926,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "8",
},
},
@ -936,7 +937,7 @@ func Test_validateUtil_Validate(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "8",
},
},

9
internal/querynodev2/delegator/delegator_data_test.go
View File

@ -36,6 +36,7 @@ import (
"github.com/milvus-io/milvus/internal/querynodev2/segments"
"github.com/milvus-io/milvus/internal/querynodev2/tsafe"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
@ -90,7 +91,7 @@ func (s *DelegatorDataSuite) SetupTest() {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -105,17 +106,17 @@ func (s *DelegatorDataSuite) SetupTest() {
IndexName: "binary_index",
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "BIN_IVF_FLAT",
},
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "TANIMOTO",
},
},

9
internal/querynodev2/delegator/delegator_test.go
View File

@ -38,6 +38,7 @@ import (
"github.com/milvus-io/milvus/internal/querynodev2/cluster"
"github.com/milvus-io/milvus/internal/querynodev2/segments"
"github.com/milvus-io/milvus/internal/querynodev2/tsafe"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/merr"
@ -110,7 +111,7 @@ func (s *DelegatorSuite) SetupTest() {
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -125,17 +126,17 @@ func (s *DelegatorSuite) SetupTest() {
IndexName: "binary_index",
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "BIN_IVF_FLAT",
},
{
Key: "metric_type",
Key: common.MetricTypeKey,
Value: "TANIMOTO",
},
},

12
internal/querynodev2/segments/mock_data.go
View File

@ -81,7 +81,7 @@ const (
defaultDim = 128
defaultMetricType = "L2"
dimKey = "dim"
dimKey = common.DimKey
defaultLocalStorage = "/tmp/milvus_test/querynode"
)
@ -211,7 +211,7 @@ func genPKFieldSchema(param constFieldParam) *schemapb.FieldSchema {
if param.dataType == schemapb.DataType_VarChar {
field.TypeParams = []*commonpb.KeyValuePair{
{Key: "max_length", Value: "12"},
{Key: common.MaxLengthKey, Value: "12"},
}
}
return field
@ -890,11 +890,11 @@ func GenAndSaveIndex(collectionID, partitionID, segmentID, fieldID int64, msgLen
func genIndexParams(indexType, metricType string) (map[string]string, map[string]string) {
typeParams := make(map[string]string)
typeParams["dim"] = strconv.Itoa(defaultDim)
typeParams[common.DimKey] = strconv.Itoa(defaultDim)
indexParams := make(map[string]string)
indexParams["index_type"] = indexType
indexParams["metric_type"] = metricType
indexParams[common.IndexTypeKey] = indexType
indexParams[common.MetricTypeKey] = metricType
indexParams["index_mode"] = "cpu"
if indexType == IndexFaissIDMap { // float vector
} else if indexType == IndexFaissIVFFlat {
@ -915,7 +915,7 @@ func genIndexParams(indexType, metricType string) (map[string]string, map[string
indexParams["m"] = strconv.Itoa(m)
indexParams["nbits"] = strconv.Itoa(nbits)
} else if indexType == IndexFaissBinIDMap {
//indexParams["dim"] = strconv.Itoa(defaultDim)
//indexParams[common.DimKey] = strconv.Itoa(defaultDim)
} else {
panic("")
}

4
internal/querynodev2/segments/segment_loader.go
View File

@ -681,7 +681,7 @@ func (loader *segmentLoader) loadFieldIndex(ctx context.Context, segment *LocalS
indexParams := funcutil.KeyValuePair2Map(indexInfo.IndexParams)
// load on disk index
if indexParams["index_type"] == indexparamcheck.IndexDISKANN {
if indexParams[common.IndexTypeKey] == indexparamcheck.IndexDISKANN {
return segment.LoadIndex(nil, indexInfo, fieldType)
}
// load in memory index
@ -929,7 +929,7 @@ func JoinIDPath(ids ...UniqueID) string {
}
func GetStorageSizeByIndexInfo(indexInfo *querypb.FieldIndexInfo) (uint64, uint64, error) {
indexType, err := funcutil.GetAttrByKeyFromRepeatedKV("index_type", indexInfo.IndexParams)
indexType, err := funcutil.GetAttrByKeyFromRepeatedKV(common.IndexTypeKey, indexInfo.IndexParams)
if err != nil {
return 0, 0, fmt.Errorf("index type not exist in index params")
}

4
internal/querynodev2/segments/utils.go
View File

@ -61,7 +61,7 @@ func getPKsFromRowBasedInsertMsg(msg *msgstream.InsertMsg, schema *schemapb.Coll
offset += 8
case schemapb.DataType_FloatVector:
for _, t := range field.TypeParams {
if t.Key == "dim" {
if t.Key == common.DimKey {
dim, err := strconv.Atoi(t.Value)
if err != nil {
return nil, fmt.Errorf("strconv wrong on get dim, err = %s", err)
@ -72,7 +72,7 @@ func getPKsFromRowBasedInsertMsg(msg *msgstream.InsertMsg, schema *schemapb.Coll
}
case schemapb.DataType_BinaryVector:
for _, t := range field.TypeParams {
if t.Key == "dim" {
if t.Key == common.DimKey {
dim, err := strconv.Atoi(t.Value)
if err != nil {
return nil, fmt.Errorf("strconv wrong on get dim, err = %s", err)

5
internal/storage/index_data_codec_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/uniquegenerator"
)
@ -35,7 +36,7 @@ func TestIndexFileBinlogCodec(t *testing.T) {
indexName := funcutil.GenRandomStr()
indexID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
indexParams := make(map[string]string)
indexParams["index_type"] = "IVF_FLAT"
indexParams[common.IndexTypeKey] = "IVF_FLAT"
datas := []*Blob{
{
Key: "ivf1",
@ -103,7 +104,7 @@ func TestIndexFileBinlogCodecError(t *testing.T) {
indexName := funcutil.GenRandomStr()
indexID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
indexParams := make(map[string]string)
indexParams["index_type"] = "IVF_FLAT"
indexParams[common.IndexTypeKey] = "IVF_FLAT"
datas := []*Blob{
{
Key: "ivf1",

3
internal/storage/print_binlog_test.go
View File

@ -24,6 +24,7 @@ import (
"time"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/stretchr/testify/assert"
@ -429,7 +430,7 @@ func TestPrintIndexFile(t *testing.T) {
indexName := funcutil.GenRandomStr()
indexID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
indexParams := make(map[string]string)
indexParams["index_type"] = "IVF_FLAT"
indexParams[common.IndexTypeKey] = "IVF_FLAT"
datas := []*Blob{
{
Key: "ivf1",

2
internal/storage/utils.go
View File

@ -178,7 +178,7 @@ func GetDimFromParams(params []*commonpb.KeyValuePair) (int, error) {
var dim int
var err error
for _, t := range params {
if t.Key == "dim" {
if t.Key == common.DimKey {
dim, err = strconv.Atoi(t.Value)
if err != nil {
return -1, err

8
internal/storage/utils_test.go
View File

@ -239,7 +239,7 @@ func TestGetDimFromParams(t *testing.T) {
dim := 8
params1 := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
}
@ -249,7 +249,7 @@ func TestGetDimFromParams(t *testing.T) {
params2 := []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "not in int format",
},
}
@ -345,7 +345,7 @@ func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(fVecDim),
},
},
@ -354,7 +354,7 @@ func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(bVecDim),
},
},

2
internal/util/importutil/import_util.go
View File

@ -336,7 +336,7 @@ func GetFileNameAndExt(filePath string) (string, string) {
func getFieldDimension(schema *schemapb.FieldSchema) (int, error) {
for _, kvPair := range schema.GetTypeParams() {
key, value := kvPair.GetKey(), kvPair.GetValue()
if key == "dim" {
if key == common.DimKey {
dim, err := strconv.Atoi(value)
if err != nil {
return 0, fmt.Errorf("illegal vector dimension '%s' for field '%s', error: %w", value, schema.GetName(), err)

21
internal/util/importutil/import_util_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/stretchr/testify/assert"
)
@ -94,7 +95,7 @@ func sampleSchema() *schemapb.CollectionSchema {
Description: "string",
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{Key: "max_length", Value: "128"},
{Key: common.MaxLengthKey, Value: "128"},
},
},
{
@ -104,7 +105,7 @@ func sampleSchema() *schemapb.CollectionSchema {
Description: "binary_vector",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "16"},
{Key: common.DimKey, Value: "16"},
},
},
{
@ -114,7 +115,7 @@ func sampleSchema() *schemapb.CollectionSchema {
Description: "float_vector",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "4"},
{Key: common.DimKey, Value: "4"},
},
},
},
@ -154,7 +155,7 @@ func strKeySchema() *schemapb.CollectionSchema {
Description: "uid",
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{Key: "max_length", Value: "1024"},
{Key: common.MaxLengthKey, Value: "1024"},
},
},
{
@ -178,7 +179,7 @@ func strKeySchema() *schemapb.CollectionSchema {
Description: "string_scalar",
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{Key: "max_length", Value: "128"},
{Key: common.MaxLengthKey, Value: "128"},
},
},
{
@ -195,7 +196,7 @@ func strKeySchema() *schemapb.CollectionSchema {
Description: "vectors",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "4"},
{Key: common.DimKey, Value: "4"},
},
},
},
@ -421,7 +422,7 @@ func Test_InitValidators(t *testing.T) {
IsPrimaryKey: false,
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "aa"},
{Key: common.DimKey, Value: "aa"},
},
})
@ -436,7 +437,7 @@ func Test_InitValidators(t *testing.T) {
IsPrimaryKey: false,
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "aa"},
{Key: common.DimKey, Value: "aa"},
},
})
@ -472,7 +473,7 @@ func Test_GetFieldDimension(t *testing.T) {
Description: "float_vector",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "4"},
{Key: common.DimKey, Value: "4"},
},
}
@ -481,7 +482,7 @@ func Test_GetFieldDimension(t *testing.T) {
assert.Equal(t, 4, dim)
schema.TypeParams = []*commonpb.KeyValuePair{
{Key: "dim", Value: "abc"},
{Key: common.DimKey, Value: "abc"},
}
dim, err = getFieldDimension(schema)
assert.NotNil(t, err)

2
internal/util/importutil/import_wrapper_test.go
View File

@ -475,7 +475,7 @@ func perfSchema(dim int) *schemapb.CollectionSchema {
Description: "float_vector",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: strconv.Itoa(dim)},
{Key: common.DimKey, Value: strconv.Itoa(dim)},
},
},
},

3
internal/util/importutil/json_handler_test.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
)
type mockIDAllocator struct {
@ -280,7 +281,7 @@ func Test_JSONRowConsumerHandle(t *testing.T) {
AutoID: true,
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{Key: "max_length", Value: "1024"},
{Key: common.MaxLengthKey, Value: "1024"},
},
},
},

3
internal/util/importutil/json_parser_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/stretchr/testify/assert"
)
@ -62,7 +63,7 @@ func Test_AdjustBufSize(t *testing.T) {
// huge row
schema.Fields[9].TypeParams = []*commonpb.KeyValuePair{
{Key: "dim", Value: "32768"},
{Key: common.DimKey, Value: "32768"},
}
parser = NewJSONParser(ctx, schema, nil)
assert.NotNil(t, parser)

5
internal/util/importutil/numpy_parser_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/timerecord"
)
@ -187,7 +188,7 @@ func Test_NumpyParserValidateHeader(t *testing.T) {
Description: "float_vector",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "99"},
{Key: common.DimKey, Value: "99"},
},
}
err = validateHeader(data3, schema)
@ -215,7 +216,7 @@ func Test_NumpyParserValidateHeader(t *testing.T) {
Description: "binary_vector",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: "dim", Value: "99"},
{Key: common.DimKey, Value: "99"},
},
}
err = validateHeader(data3, schema)

39
internal/util/indexcgowrapper/codec_index_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
)
@ -161,14 +162,14 @@ func genScalarIndexCases(dtype schemapb.DataType) []indexTestCase {
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": "inverted_index",
common.IndexTypeKey: "inverted_index",
},
},
{
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": "flat",
common.IndexTypeKey: "flat",
},
},
}
@ -180,14 +181,14 @@ func genStringIndexCases(dtype schemapb.DataType) []indexTestCase {
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": "inverted_index",
common.IndexTypeKey: "inverted_index",
},
},
{
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": "marisa-trie",
common.IndexTypeKey: "marisa-trie",
},
},
}
@ -199,22 +200,22 @@ func genFloatVecIndexCases(dtype schemapb.DataType) []indexTestCase {
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": IndexFaissIVFPQ,
"metric_type": L2,
"dim": strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"m": strconv.Itoa(m),
"nbits": strconv.Itoa(nbits),
common.IndexTypeKey: IndexFaissIVFPQ,
common.MetricTypeKey: L2,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"m": strconv.Itoa(m),
"nbits": strconv.Itoa(nbits),
},
},
{
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": IndexFaissIVFFlat,
"metric_type": L2,
"dim": strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
common.IndexTypeKey: IndexFaissIVFFlat,
common.MetricTypeKey: L2,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
},
},
}
@ -226,11 +227,11 @@ func genBinaryVecIndexCases(dtype schemapb.DataType) []indexTestCase {
dtype: dtype,
typeParams: nil,
indexParams: map[string]string{
"index_type": IndexFaissBinIVFFlat,
"metric_type": Jaccard,
"dim": strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"nbits": strconv.Itoa(nbits),
common.IndexTypeKey: IndexFaissBinIVFFlat,
common.MetricTypeKey: Jaccard,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"nbits": strconv.Itoa(nbits),
},
},
}

23
internal/util/indexcgowrapper/index_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -79,34 +80,34 @@ func generateTestCases() []vecTestCase {
func generateParams(indexType, metricType string) (map[string]string, map[string]string) {
typeParams := make(map[string]string)
indexParams := make(map[string]string)
indexParams["index_type"] = indexType
indexParams["metric_type"] = metricType
indexParams[common.IndexTypeKey] = indexType
indexParams[common.MetricTypeKey] = metricType
if indexType == IndexFaissIDMap { // float vector
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
} else if indexType == IndexFaissIVFFlat {
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
} else if indexType == IndexFaissIVFPQ {
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["m"] = strconv.Itoa(m)
indexParams["nbits"] = strconv.Itoa(nbits)
} else if indexType == IndexFaissIVFSQ8 {
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["nbits"] = strconv.Itoa(nbits)
} else if indexType == IndexHNSW {
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["M"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
indexParams["ef"] = strconv.Itoa(ef)
} else if indexType == IndexFaissBinIVFFlat { // binary vector
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["m"] = strconv.Itoa(m)
indexParams["nbits"] = strconv.Itoa(nbits)
} else if indexType == IndexFaissBinIDMap {
indexParams["dim"] = strconv.Itoa(dim)
indexParams[common.DimKey] = strconv.Itoa(dim)
} else {
panic("")
}
@ -215,8 +216,8 @@ func TestCIndex_Delete(t *testing.T) {
func TestCIndex_Error(t *testing.T) {
indexParams := make(map[string]string)
indexParams["index_type"] = "IVF_FLAT"
indexParams["metric_type"] = "L2"
indexParams[common.IndexTypeKey] = "IVF_FLAT"
indexParams[common.MetricTypeKey] = "L2"
indexPtr, err := NewCgoIndex(schemapb.DataType_FloatVector, nil, indexParams, genStorageConfig())
assert.Nil(t, err)

3
internal/util/typeutil/result_helper_test.go
View File

@ -4,6 +4,7 @@ import (
"testing"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -92,7 +93,7 @@ func TestGenEmptyFieldData(t *testing.T) {
assert.Error(t, err)
}
field.TypeParams = []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}
field.TypeParams = []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "128"}}
for _, dataType := range vectorTypes {
field.DataType = dataType
fieldData, err := typeutil.GenEmptyFieldData(field)

1
pkg/common/common.go
View File

@ -85,6 +85,7 @@ const (
IndexTypeKey = "index_type"
MetricTypeKey = "metric_type"
DimKey = "dim"
MaxLengthKey = "max_length"
)
// Collection properties key

3
pkg/util/constant.go
View File

@ -20,6 +20,7 @@ import (
"strings"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/pkg/common"
)
// Meta Prefix consts
@ -58,7 +59,7 @@ const (
const (
// ParamsKeyToParse is the key of the param to build index.
ParamsKeyToParse = "params"
ParamsKeyToParse = common.IndexParamsKey
)
var (

6
pkg/util/indexparamcheck/constraints.go
View File

@ -1,5 +1,7 @@
package indexparamcheck
import "github.com/milvus-io/milvus/pkg/common"
const (
// L2 represents Euclidean distance
L2 = "L2"
@ -50,9 +52,9 @@ const (
HNSWMaxM = 64
// DIM is a constant used to represent dimension
DIM = "dim"
DIM = common.DimKey
// Metric is a constant used to metric type
Metric = "metric_type"
Metric = common.MetricTypeKey
// NLIST is a constant used to nlist in Index IVFxxx
NLIST = "nlist"
NBITS = "nbits"

5
pkg/util/indexparams/disk_index_params.go
View File

@ -22,6 +22,7 @@ import (
"strconv"
"unsafe"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/paramtable"
@ -182,7 +183,7 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
// SetDiskIndexBuildParams set index build params with ratio params on indexNode
// IndexNode cal build param with ratio params and cpu count, memory count...
func SetDiskIndexBuildParams(indexParams map[string]string, numRows int64) error {
dimStr, ok := indexParams["dim"]
dimStr, ok := indexParams[common.DimKey]
if !ok {
// type param dim has been put into index params before build index
return fmt.Errorf("type param dim not exist")
@ -220,7 +221,7 @@ func SetDiskIndexBuildParams(indexParams map[string]string, numRows int64) error
// SetDiskIndexLoadParams set disk index load params with ratio params on queryNode
// QueryNode cal load params with ratio params ans cpu count...
func SetDiskIndexLoadParams(params *paramtable.ComponentParam, indexParams map[string]string, numRows int64) error {
dimStr, ok := indexParams["dim"]
dimStr, ok := indexParams[common.DimKey]
if !ok {
// type param dim has been put into index params before build index
return fmt.Errorf("type param dim not exist")

19
pkg/util/indexparams/disk_index_params_test.go
View File

@ -22,6 +22,7 @@ import (
"strconv"
"testing"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/stretchr/testify/assert"
@ -60,7 +61,7 @@ func TestDiskIndexParams(t *testing.T) {
indexParams := make(map[string]string)
indexParams["max_degree"] = "56"
indexParams["search_list_size"] = "100"
indexParams["index_type"] = "DISKANN"
indexParams[common.IndexTypeKey] = "DISKANN"
str, err = json.Marshal(indexParams)
assert.NoError(t, err)
params.Save(params.AutoIndexConfig.IndexParams.Key, string(str))
@ -87,7 +88,7 @@ func TestDiskIndexParams(t *testing.T) {
indexParams := make(map[string]string)
indexParams["max_degree"] = "56"
indexParams["search_list_size"] = "100"
indexParams["index_type"] = "DISKANN"
indexParams[common.IndexTypeKey] = "DISKANN"
str, err := json.Marshal(indexParams)
assert.NoError(t, err)
params.Save(params.AutoIndexConfig.IndexParams.Key, string(str))
@ -108,7 +109,7 @@ func TestDiskIndexParams(t *testing.T) {
indexParams = make(map[string]string)
indexParams["max_degree"] = "56"
indexParams["search_list"] = "100" // should be search_list_size
indexParams["index_type"] = "DISKANN"
indexParams[common.IndexTypeKey] = "DISKANN"
str, err = json.Marshal(indexParams)
assert.NoError(t, err)
params.Save(params.AutoIndexConfig.IndexParams.Key, string(str))
@ -126,7 +127,7 @@ func TestDiskIndexParams(t *testing.T) {
err := SetDiskIndexBuildParams(indexParams, 100)
assert.Error(t, err)
indexParams["dim"] = "128"
indexParams[common.DimKey] = "128"
err = SetDiskIndexBuildParams(indexParams, 100)
assert.NoError(t, err)
@ -146,7 +147,7 @@ func TestDiskIndexParams(t *testing.T) {
err := SetDiskIndexLoadParams(&params, indexParams, 100)
assert.Error(t, err)
indexParams["dim"] = "128"
indexParams[common.DimKey] = "128"
err = SetDiskIndexLoadParams(&params, indexParams, 100)
assert.NoError(t, err)
@ -209,7 +210,7 @@ func TestDiskIndexParams(t *testing.T) {
err = SetDiskIndexLoadParams(&params, indexParams, 100)
assert.Error(t, err)
indexParams["dim"] = "128"
indexParams[common.DimKey] = "128"
err = SetDiskIndexLoadParams(&params, indexParams, 100)
assert.NoError(t, err)
@ -243,11 +244,11 @@ func TestDiskIndexParams(t *testing.T) {
indexParams := make(map[string]string)
indexParams["max_degree"] = "56"
indexParams["search_list_size"] = "100"
indexParams["index_type"] = "DISKANN"
indexParams[common.IndexTypeKey] = "DISKANN"
str, err := json.Marshal(indexParams)
assert.NoError(t, err)
params.Save(params.AutoIndexConfig.IndexParams.Key, string(str))
indexParams["dim"] = "128"
indexParams[common.DimKey] = "128"
err = SetDiskIndexLoadParams(&params, indexParams, 100)
assert.Error(t, err)
@ -268,7 +269,7 @@ func TestDiskIndexParams(t *testing.T) {
indexParams = make(map[string]string)
indexParams["max_degree"] = "56"
indexParams["search_list"] = "100" // should be search_list_size
indexParams["index_type"] = "DISKANN"
indexParams[common.IndexTypeKey] = "DISKANN"
str, err = json.Marshal(indexParams)
assert.NoError(t, err)
params.Save(params.AutoIndexConfig.IndexParams.Key, string(str))

3
pkg/util/typeutil/get_dim.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/common"
)
// GetDim get dimension of field. Maybe also helpful outside.
@ -13,7 +14,7 @@ func GetDim(field *schemapb.FieldSchema) (int64, error) {
return 0, fmt.Errorf("%s is not of vector type", field.GetDataType())
}
h := NewKvPairs(append(field.GetIndexParams(), field.GetTypeParams()...))
dimStr, err := h.Get("dim")
dimStr, err := h.Get(common.DimKey)
if err != nil {
return 0, fmt.Errorf("dim not found")
}

5
pkg/util/typeutil/kv_pair_helper_test.go
View File

@ -4,15 +4,16 @@ import (
"testing"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/stretchr/testify/assert"
)
func TestNewKvPairs(t *testing.T) {
kvPairs := []*commonpb.KeyValuePair{
{Key: "dim", Value: "128"},
{Key: common.DimKey, Value: "128"},
}
h := NewKvPairs(kvPairs)
v, err := h.Get("dim")
v, err := h.Get(common.DimKey)
assert.NoError(t, err)
assert.Equal(t, "128", v)
_, err = h.Get("not_exist")

13
pkg/util/typeutil/schema.go
View File

@ -23,8 +23,10 @@ import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/log"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
)
const DynamicFieldMaxLength = 512
@ -40,8 +42,7 @@ func GetAvgLengthOfVarLengthField(fieldSchema *schemapb.FieldSchema) (int, error
switch fieldSchema.DataType {
case schemapb.DataType_VarChar:
maxLengthPerRowKey := "max_length"
maxLengthPerRowValue, ok := paramsMap[maxLengthPerRowKey]
maxLengthPerRowValue, ok := paramsMap[common.MaxLengthKey]
if !ok {
return 0, fmt.Errorf("the max_length was not specified, field type is %s", fieldSchema.DataType.String())
}
@ -84,7 +85,7 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
res += maxLengthPerRow
case schemapb.DataType_BinaryVector:
for _, kv := range fs.TypeParams {
if kv.Key == "dim" {
if kv.Key == common.DimKey {
v, err := strconv.Atoi(kv.Value)
if err != nil {
return -1, err
@ -95,7 +96,7 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
}
case schemapb.DataType_FloatVector:
for _, kv := range fs.TypeParams {
if kv.Key == "dim" {
if kv.Key == common.DimKey {
v, err := strconv.Atoi(kv.Value)
if err != nil {
return -1, err
@ -286,7 +287,7 @@ func (helper *SchemaHelper) GetVectorDimFromID(fieldID int64) (int, error) {
return 0, fmt.Errorf("field type = %s not has dim", schemapb.DataType_name[int32(sch.DataType)])
}
for _, kv := range sch.TypeParams {
if kv.Key == "dim" {
if kv.Key == common.DimKey {
dim, err := strconv.Atoi(kv.Value)
if err != nil {
return 0, err

8
pkg/util/typeutil/schema_test.go
View File

@ -87,7 +87,7 @@ func TestSchema(t *testing.T) {
DataType: 21,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "125",
},
},
@ -100,7 +100,7 @@ func TestSchema(t *testing.T) {
DataType: 101,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -119,7 +119,7 @@ func TestSchema(t *testing.T) {
DataType: 100,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: "128",
},
},
@ -957,7 +957,7 @@ func TestCalcColumnSize(t *testing.T) {
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "125",
},
},

12
tests/integration/bulkinsert_test.go
View File

@ -71,8 +71,8 @@ func TestBulkInsert(t *testing.T) {
schema := constructSchema(collectionName, dim, true,
&schemapb.FieldSchema{Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true, AutoID: true},
&schemapb.FieldSchema{Name: "image_path", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "max_length", Value: "65535"}}},
&schemapb.FieldSchema{Name: "embeddings", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
&schemapb.FieldSchema{Name: "image_path", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: common.MaxLengthKey, Value: "65535"}}},
&schemapb.FieldSchema{Name: "embeddings", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "128"}}},
)
marshaledSchema, err := proto.Marshal(schema)
assert.NoError(t, err)
@ -98,14 +98,14 @@ func TestBulkInsert(t *testing.T) {
err = GenerateNumpyFile(c.chunkManager.RootPath()+"/"+"embeddings.npy", 100, schemapb.DataType_FloatVector, []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(Dim),
},
})
assert.NoError(t, err)
err = GenerateNumpyFile(c.chunkManager.RootPath()+"/"+"image_path.npy", 100, schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: strconv.Itoa(65535),
},
})
@ -225,7 +225,7 @@ func GenerateNumpyFile(filePath string, rowCount int, dType schemapb.DataType, t
}
}
if dType == schemapb.DataType_FloatVector {
dimStr, ok := funcutil.KeyValuePair2Map(typeParams)["dim"]
dimStr, ok := funcutil.KeyValuePair2Map(typeParams)[common.DimKey]
if !ok {
return errors.New("FloatVector field needs dim parameter")
}
@ -260,7 +260,7 @@ func TestGenerateNumpyFile(t *testing.T) {
require.NoError(t, err)
err = GenerateNumpyFile(TempFilesPath+"embeddings.npy", 100, schemapb.DataType_FloatVector, []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(Dim),
},
})

16
tests/integration/get_vector_test.go
View File

@ -51,7 +51,7 @@ type TestGetVectorSuite struct {
vecType schemapb.DataType
}
func (suite *TestGetVectorSuite) SetupSuite() {
func (suite *TestGetVectorSuite) SetupTest() {
suite.ctx, suite.cancel = context.WithTimeout(context.Background(), time.Second*180)
var err error
@ -61,6 +61,12 @@ func (suite *TestGetVectorSuite) SetupSuite() {
suite.Require().NoError(err)
}
func (suite *TestGetVectorSuite) TearDownTest() {
err := suite.cluster.Stop()
suite.Require().NoError(err)
suite.cancel()
}
func (suite *TestGetVectorSuite) run() {
collection := fmt.Sprintf("TestGetVector_%d_%d_%s_%s_%s",
suite.nq, suite.topK, suite.indexType, suite.metricType, funcutil.GenRandomStr())
@ -80,7 +86,7 @@ func (suite *TestGetVectorSuite) run() {
DataType: suite.pkType,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Key: common.MaxLengthKey,
Value: "100",
},
},
@ -358,12 +364,6 @@ func (suite *TestGetVectorSuite) TestGetVector_Big_NQ_TOPK() {
// suite.run()
//}
func (suite *TestGetVectorSuite) TearDownSuite() {
err := suite.cluster.Stop()
suite.Require().NoError(err)
suite.cancel()
}
func TestGetVector(t *testing.T) {
suite.Run(t, new(TestGetVectorSuite))
}

6
tests/integration/json_expr_test.go
View File

@ -73,7 +73,7 @@ func TestJsonExpr(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -178,7 +178,7 @@ func TestJsonExpr(t *testing.T) {
IndexName: "_default",
ExtraParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
{
@ -186,7 +186,7 @@ func TestJsonExpr(t *testing.T) {
Value: distance.L2,
},
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{

8
tests/integration/meta_watcher_test.go
View File

@ -90,7 +90,7 @@ func TestShowSegments(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -182,7 +182,7 @@ func TestShowReplicas(t *testing.T) {
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
@ -279,7 +279,7 @@ func TestShowReplicas(t *testing.T) {
IndexName: "_default",
ExtraParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
{
@ -287,7 +287,7 @@ func TestShowReplicas(t *testing.T) {
Value: distance.L2,
},
{
Key: "index_type",
Key: common.IndexTypeKey,
Value: "IVF_FLAT",
},
{

2
tests/integration/util_index.go
View File

@ -130,7 +130,7 @@ func getSearchParams(indexType string, metricType string) map[string]any {
params := make(map[string]any)
switch indexType {
case IndexFaissIDMap, IndexFaissBinIDMap:
params["metric_type"] = metricType
params[common.MetricTypeKey] = metricType
case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexFaissIvfPQ:
params["nprobe"] = 8
case IndexHNSW:

4
tests/integration/util_query.go
View File

@ -37,8 +37,8 @@ const (
AnnsFieldKey = "anns_field"
TopKKey = "topk"
NQKey = "nq"
MetricTypeKey = "metric_type"
SearchParamsKey = "params"
MetricTypeKey = common.MetricTypeKey
SearchParamsKey = common.IndexParamsKey
RoundDecimalKey = "round_decimal"
OffsetKey = "offset"
LimitKey = "limit"