Remove hardcode max shards num (#22004)

Signed-off-by: longjiquan <jiquan.long@zilliz.com>

configs/milvus.yaml

@@ -161,7 +161,8 @@ proxy:
   # As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64.
   # So adjust at your risk!
   maxDimension: 32768 # Maximum dimension of a vector
-  maxShardNum: 256 # Maximum number of shards in a collection
+  # It is strongly DISCOURAGED to set `maxShardNum` > 64.
+  maxShardNum: 64 # Maximum number of shards in a collection
   maxTaskNum: 1024 # max task number of proxy task queue
   # please adjust in embedded Milvus: false
   ginLogging: true # Whether to produce gin logs.

internal/rootcoord/create_collection_task.go

@@ -51,8 +51,16 @@ func (t *createCollectionTask) validate() error {
 		return err
 	}
 
-	if t.Req.GetShardsNum() >= maxShardNum {
-		return fmt.Errorf("shard num (%d) exceeds limit (%d)", t.Req.GetShardsNum(), maxShardNum)
+	shardsNum := t.Req.GetShardsNum()
+
+	cfgMaxShardNum := Params.RootCoordCfg.DmlChannelNum.GetAsInt32()
+	if shardsNum > cfgMaxShardNum {
+		return fmt.Errorf("shard num (%d) exceeds max configuration (%d)", shardsNum, cfgMaxShardNum)
+	}
+
+	cfgShardLimit := Params.ProxyCfg.MaxShardNum.GetAsInt32()
+	if shardsNum > cfgShardLimit {
+		return fmt.Errorf("shard num (%d) exceeds system limit (%d)", shardsNum, cfgShardLimit)
 	}
 
 	return nil

internal/rootcoord/create_collection_task_test.go

@@ -40,11 +40,26 @@ func Test_createCollectionTask_validate(t *testing.T) {
 		assert.Error(t, err)
 	})
 
-	t.Run("shard num exceeds limit", func(t *testing.T) {
+	t.Run("shard num exceeds max configuration", func(t *testing.T) {
+		// TODO: better to have a `Set` method for ParamItem.
+		cfgMaxShardNum := Params.RootCoordCfg.DmlChannelNum.GetAsInt32()
 		task := createCollectionTask{
 			Req: &milvuspb.CreateCollectionRequest{
 				Base:      &commonpb.MsgBase{MsgType: commonpb.MsgType_CreateCollection},
-				ShardsNum: maxShardNum + 1,
+				ShardsNum: cfgMaxShardNum + 1,
+			},
+		}
+		err := task.validate()
+		assert.Error(t, err)
+	})
+
+	t.Run("shard num exceeds limit", func(t *testing.T) {
+		// TODO: better to have a `Set` method for ParamItem.
+		cfgShardLimit := Params.ProxyCfg.MaxShardNum.GetAsInt32()
+		task := createCollectionTask{
+			Req: &milvuspb.CreateCollectionRequest{
+				Base:      &commonpb.MsgBase{MsgType: commonpb.MsgType_CreateCollection},
+				ShardsNum: cfgShardLimit + 1,
 			},
 		}
 		err := task.validate()