milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: add addition index params for raft index (#30179) 

issue: https://github.com/milvus-io/milvus/issues/29230

Signed-off-by: yusheng.ma <yusheng.ma@zilliz.com>
Co-authored-by: yusheng.ma <yusheng.ma@zilliz.com>
pull/30753/head^2
cqy123456 2024-02-26 14:50:54 +08:00 committed by GitHub
parent a0531b72aa
commit b31d1a1eb5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 92 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/util/indexparamcheck/cagra_checker.go
View File

@ -35,8 +35,20 @@ func (c *cagraChecker) CheckTrain(params map[string]string) error {
return fmt.Errorf("Graph degree cannot be larger than intermediate graph degree")
}
if !CheckStrByValues(params, Metric, CagraMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", CagraMetrics)
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
setDefaultIfNotExist(params, CagraBuildAlgo, "NN_DESCENT")
if !CheckStrByValues(params, CagraBuildAlgo, CagraBuildAlgoTypes) {
return fmt.Errorf("cagra build algo type not supported, supported: %v", CagraBuildAlgoTypes)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil

27
pkg/util/indexparamcheck/cagra_checker_test.go
View File

@ -61,13 +61,32 @@ func Test_cagraChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(0),
Metric: metric.L2,
}
p11 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
CagraBuildAlgo: "IVF_PQ",
}
p12 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
CagraBuildAlgo: "HNSW",
}
p13 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
RaftCacheDatasetOnDevice: "false",
}
p14 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
RaftCacheDatasetOnDevice: "False",
}
cases := []struct {
params map[string]string
errIsNil bool
}{
{p1, true},
{p2, false},
{p2, true},
{p3, true},
{p4, true},
{p5, true},
@ -76,6 +95,10 @@ func Test_cagraChecker_CheckTrain(t *testing.T) {
{p8, false},
{p9, false},
{p10, false},
{p11, true},
{p12, false},
{p13, true},
{p14, false},
}
c := newCagraChecker()

8
pkg/util/indexparamcheck/constraints.go
View File

@ -37,9 +37,15 @@ const (
EFConstruction = "efConstruction"
HNSWM = "M"
RaftCacheDatasetOnDevice = "cache_dataset_on_device"
// Cagra Train Param
CagraInterDegree = "intermediate_graph_degree"
CagraGraphDegree = "graph_degree"
CagraBuildAlgo = "build_algo"
CargaBuildAlgoIVFPQ = "IVF_PQ"
CargaBuildAlgoNNDESCENT = "NN_DESCENT"
)
// METRICS is a set of all metrics types supported for float vector.
@ -51,7 +57,7 @@ var (
BinIvfMetrics = []string{metric.HAMMING, metric.JACCARD} // const
HnswMetrics = []string{metric.L2, metric.IP, metric.COSINE, metric.HAMMING, metric.JACCARD} // const
RaftMetrics = []string{metric.L2, metric.IP}
CagraMetrics = []string{metric.L2} // const
CagraBuildAlgoTypes = []string{CargaBuildAlgoIVFPQ, CargaBuildAlgoNNDESCENT}
supportDimPerSubQuantizer = []int{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1} // const
supportSubQuantizer = []int{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1} // const
)

7
pkg/util/indexparamcheck/raft_ivf_flat_checker.go
View File

@ -15,6 +15,13 @@ func (c *raftIVFFlatChecker) CheckTrain(params map[string]string) error {
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil
}

14
pkg/util/indexparamcheck/raft_ivf_flat_checker_test.go
View File

@ -53,6 +53,18 @@ func Test_raftIvfFlatChecker_CheckTrain(t *testing.T) {
NLIST: strconv.Itoa(1024),
Metric: metric.SUPERSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
NLIST: strconv.Itoa(1024),
RaftCacheDatasetOnDevice: "false",
}
p9 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.L2,
NLIST: strconv.Itoa(1024),
RaftCacheDatasetOnDevice: "False",
}
cases := []struct {
params map[string]string
@ -68,6 +80,8 @@ func Test_raftIvfFlatChecker_CheckTrain(t *testing.T) {
{p5, false},
{p6, false},
{p7, false},
{p8, true},
{p9, false},
}
c := newRaftIVFFlatChecker()

7
pkg/util/indexparamcheck/raft_ivf_pq_checker.go
View File

@ -57,6 +57,13 @@ func (c *raftIVFPQChecker) checkPQParams(params map[string]string) error {
if dimension%m != 0 {
return fmt.Errorf("dimension must be able to be divided by `m`, dimension: %d, m: %d", dimension, m)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil
}

18
pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go
View File

@ -105,6 +105,22 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.SUPERSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.L2,
RaftCacheDatasetOnDevice: "false",
}
p9 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.L2,
RaftCacheDatasetOnDevice: "False",
}
cases := []struct {
params map[string]string
@ -128,6 +144,8 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
{p5, false},
{p6, false},
{p7, false},
{p8, true},
{p9, false},
}
c := newRaftIVFPQChecker()