milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.5] add param for tuning max VARCHAR length and restore limit to 65535 (#38883) 

issue: #38882
pr: https://github.com/milvus-io/milvus/pull/38884

Signed-off-by: Patrick Weizhi Xu <weizhi.xu@zilliz.com>
(cherry picked from commit a64a7377ea8258efe36c74bd1cf9db556726ba2d)
pull/38932/head
Patrick Weizhi Xu 2025-01-02 13:56:54 +08:00 committed by GitHub
parent a2c4cd59ce
commit 1d55ad6a1d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/proxy/task.go
View File

@ -1256,8 +1256,9 @@ func (t *alterCollectionFieldTask) PreExecute(ctx context.Context) error {
return merr.WrapErrParameterInvalid("%s should be an integer, but got %T", prop.Key, prop.Value)
}
if value > defaultMaxVarCharLength {
return merr.WrapErrParameterInvalid("%s exceeds the maximum allowed value 1048576", prop.Value)
defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
if int64(value) > defaultMaxVarCharLength {
return merr.WrapErrParameterInvalidMsg("%s exceeds the maximum allowed value %s", prop.Value, strconv.FormatInt(defaultMaxVarCharLength, 10))
}
}
}

6
internal/proxy/util.go
View File

@ -63,8 +63,6 @@ const (
// enableMultipleVectorFields indicates whether to enable multiple vector fields.
enableMultipleVectorFields = true
defaultMaxVarCharLength = 1048576
defaultMaxArrayCapacity = 4096
defaultMaxSearchRequest = 1024
@ -363,8 +361,10 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema)
if err != nil {
return err
}
defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
if maxLengthPerRow > defaultMaxVarCharLength || maxLengthPerRow <= 0 {
return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, 1048576]")
return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, %d]", defaultMaxVarCharLength)
}
exist = true
}

9
pkg/util/paramtable/component_param.go
View File

@ -1305,6 +1305,7 @@ type proxyConfig struct {
SkipAutoIDCheck ParamItem `refreshable:"true"`
SkipPartitionKeyCheck ParamItem `refreshable:"true"`
EnablePublicPrivilege ParamItem `refreshable:"false"`
MaxVarCharLength ParamItem `refreshable:"false"`
AccessLog AccessLogConfig
@ -1717,6 +1718,14 @@ please adjust in embedded Milvus: false`,
}
p.EnablePublicPrivilege.Init(base.mgr)
p.MaxVarCharLength = ParamItem{
Key: "proxy.maxVarCharLength",
Version: "2.4.19", // hotfix
DefaultValue: strconv.Itoa(65535), // 64K
Doc: "maximum number of characters for a varchar field; this value is overridden by the value in a pre-existing schema if applicable",
}
p.MaxVarCharLength.Init(base.mgr)
p.GracefulStopTimeout = ParamItem{
Key: "proxy.gracefulStopTimeout",
Version: "2.3.7",

2
tests/go_client/common/consts.go
View File

@ -57,7 +57,7 @@ const (
DefaultRgName = "__default_resource_group"
DefaultDb = "default"
MaxDim = 32768
MaxLength = int64(1048576)
MaxLength = int64(65535)
MaxCollectionNameLen = 255
DefaultRgCapacity = 1000000
RetentionDuration = 40 // common.retentionDuration

4
tests/go_client/testcases/collection_test.go
View File

@ -836,7 +836,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
arrayVarcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
}
}
@ -858,7 +858,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) {
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
varcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
}
}

6
tests/python_client/testcases/test_collection.py
View File

@ -3875,10 +3875,10 @@ class TestCollectionString(TestcaseBase):
c_name = cf.gen_unique_str(prefix)
int_field = cf.gen_int64_field(is_primary=True)
vec_field = cf.gen_float_vec_field()
max_length = 1048576 + 1
max_length = 65535 + 1
string_field = cf.gen_string_field(max_length=max_length)
schema = cf.gen_collection_schema([int_field, string_field, vec_field])
error = {ct.err_code: 1048576, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 1048576]"}
error = {ct.err_code: 65535, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 65535]"}
self.collection_wrap.init_collection(name=c_name, schema=schema,
check_task=CheckTasks.err_res, check_items=error)
@ -4117,7 +4117,7 @@ class TestCollectionARRAY(TestcaseBase):
self.init_collection_wrap(schema=array_schema, check_task=CheckTasks.err_res,
check_items={ct.err_code: 65535,
ct.err_msg: "the maximum length specified for a VarChar "
"should be in (0, 1048576]"})
"should be in (0, 65535]"})
@pytest.mark.tags(CaseLabel.L2)
def test_collection_array_field_all_datatype(self):