enhance: add addition index params for raft index (#30179)

issue: https://github.com/milvus-io/milvus/issues/29230

Signed-off-by: yusheng.ma <yusheng.ma@zilliz.com>
Co-authored-by: yusheng.ma <yusheng.ma@zilliz.com>

pkg/util/indexparamcheck/cagra_checker.go

@@ -35,8 +35,20 @@ func (c *cagraChecker) CheckTrain(params map[string]string) error {
 		return fmt.Errorf("Graph degree cannot be larger than intermediate graph degree")
 	}
 
-	if !CheckStrByValues(params, Metric, CagraMetrics) {
+	if !CheckStrByValues(params, Metric, RaftMetrics) {
-		return fmt.Errorf("metric type not found or not supported, supported: %v", CagraMetrics)
+		return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
+	}
+
+	setDefaultIfNotExist(params, CagraBuildAlgo, "NN_DESCENT")
+
+	if !CheckStrByValues(params, CagraBuildAlgo, CagraBuildAlgoTypes) {
+		return fmt.Errorf("cagra build algo type not supported, supported: %v", CagraBuildAlgoTypes)
+	}
+
+	setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
+
+	if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
+		return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
 	}
 
 	return nil

pkg/util/indexparamcheck/cagra_checker_test.go

@@ -61,13 +61,32 @@ func Test_cagraChecker_CheckTrain(t *testing.T) {
 		DIM:    strconv.Itoa(0),
 		Metric: metric.L2,
 	}
-
+	p11 := map[string]string{
+		DIM:            strconv.Itoa(128),
+		Metric:         metric.L2,
+		CagraBuildAlgo: "IVF_PQ",
+	}
+	p12 := map[string]string{
+		DIM:            strconv.Itoa(128),
+		Metric:         metric.L2,
+		CagraBuildAlgo: "HNSW",
+	}
+	p13 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		Metric:                   metric.L2,
+		RaftCacheDatasetOnDevice: "false",
+	}
+	p14 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		Metric:                   metric.L2,
+		RaftCacheDatasetOnDevice: "False",
+	}
 	cases := []struct {
 		params   map[string]string
 		errIsNil bool
 	}{
 		{p1, true},
-		{p2, false},
+		{p2, true},
 		{p3, true},
 		{p4, true},
 		{p5, true},
@@ -76,6 +95,10 @@ func Test_cagraChecker_CheckTrain(t *testing.T) {
 		{p8, false},
 		{p9, false},
 		{p10, false},
+		{p11, true},
+		{p12, false},
+		{p13, true},
+		{p14, false},
 	}
 
 	c := newCagraChecker()

pkg/util/indexparamcheck/constraints.go

@@ -37,9 +37,15 @@ const (
 	EFConstruction = "efConstruction"
 	HNSWM          = "M"
 
+	RaftCacheDatasetOnDevice = "cache_dataset_on_device"
+
 	// Cagra Train Param
 	CagraInterDegree = "intermediate_graph_degree"
 	CagraGraphDegree = "graph_degree"
+	CagraBuildAlgo   = "build_algo"
+
+	CargaBuildAlgoIVFPQ     = "IVF_PQ"
+	CargaBuildAlgoNNDESCENT = "NN_DESCENT"
 )
 
 // METRICS is a set of all metrics types supported for float vector.
@@ -51,7 +57,7 @@ var (
 	BinIvfMetrics             = []string{metric.HAMMING, metric.JACCARD}                                             // const
 	HnswMetrics               = []string{metric.L2, metric.IP, metric.COSINE, metric.HAMMING, metric.JACCARD}        // const
 	RaftMetrics               = []string{metric.L2, metric.IP}
-	CagraMetrics              = []string{metric.L2}                                              // const
+	CagraBuildAlgoTypes       = []string{CargaBuildAlgoIVFPQ, CargaBuildAlgoNNDESCENT}
 	supportDimPerSubQuantizer = []int{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}              // const
 	supportSubQuantizer       = []int{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1} // const
 )

pkg/util/indexparamcheck/raft_ivf_flat_checker.go

@@ -15,6 +15,13 @@ func (c *raftIVFFlatChecker) CheckTrain(params map[string]string) error {
 	if !CheckStrByValues(params, Metric, RaftMetrics) {
 		return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
 	}
+
+	setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
+
+	if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
+		return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
+	}
+
 	return nil
 }
 

pkg/util/indexparamcheck/raft_ivf_flat_checker_test.go

@@ -53,6 +53,18 @@ func Test_raftIvfFlatChecker_CheckTrain(t *testing.T) {
 		NLIST:  strconv.Itoa(1024),
 		Metric: metric.SUPERSTRUCTURE,
 	}
+	p8 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		Metric:                   metric.L2,
+		NLIST:                    strconv.Itoa(1024),
+		RaftCacheDatasetOnDevice: "false",
+	}
+	p9 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		Metric:                   metric.L2,
+		NLIST:                    strconv.Itoa(1024),
+		RaftCacheDatasetOnDevice: "False",
+	}
 
 	cases := []struct {
 		params   map[string]string
@@ -68,6 +80,8 @@ func Test_raftIvfFlatChecker_CheckTrain(t *testing.T) {
 		{p5, false},
 		{p6, false},
 		{p7, false},
+		{p8, true},
+		{p9, false},
 	}
 
 	c := newRaftIVFFlatChecker()

pkg/util/indexparamcheck/raft_ivf_pq_checker.go

@@ -57,6 +57,13 @@ func (c *raftIVFPQChecker) checkPQParams(params map[string]string) error {
 	if dimension%m != 0 {
 		return fmt.Errorf("dimension must be able to be divided by `m`, dimension: %d, m: %d", dimension, m)
 	}
+
+	setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
+
+	if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
+		return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
+	}
+
 	return nil
 }
 

pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go

@@ -105,6 +105,22 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
 		NBITS:  strconv.Itoa(8),
 		Metric: metric.SUPERSTRUCTURE,
 	}
+	p8 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		NLIST:                    strconv.Itoa(1024),
+		IVFM:                     strconv.Itoa(4),
+		NBITS:                    strconv.Itoa(8),
+		Metric:                   metric.L2,
+		RaftCacheDatasetOnDevice: "false",
+	}
+	p9 := map[string]string{
+		DIM:                      strconv.Itoa(128),
+		NLIST:                    strconv.Itoa(1024),
+		IVFM:                     strconv.Itoa(4),
+		NBITS:                    strconv.Itoa(8),
+		Metric:                   metric.L2,
+		RaftCacheDatasetOnDevice: "False",
+	}
 
 	cases := []struct {
 		params   map[string]string
@@ -128,6 +144,8 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
 		{p5, false},
 		{p6, false},
 		{p7, false},
+		{p8, true},
+		{p9, false},
 	}
 
 	c := newRaftIVFPQChecker()