milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove binary metrics TANIMOTO/SUPERSTRUCTURE/SUBSTRUCTURE (#25708) 

Signed-off-by: Yudong Cai <yudong.cai@zilliz.com>
pull/25747/head
Cai Yudong 2023-07-19 16:16:58 +08:00 committed by GitHub
parent bccdef1ad7
commit 9a4761dcc7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 51 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/common/RangeSearchHelper.cpp
View File

@ -125,7 +125,7 @@ CheckRangeSearchParam(float radius,
} else {
AssertInfo(range_filter < radius,
"range_filter must be less than radius for "
"L2/HAMMING/JACCARD/TANIMOTO");
"L2/HAMMING/JACCARD");
}
}

6
internal/core/src/query/deprecated/ValidationUtil.cpp
View File

@ -362,8 +362,7 @@ ValidateIndexMetricType(const std::string_view metric_type,
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
// binary
if (metric_type != knowhere::Metric::HAMMING &&
metric_type != knowhere::Metric::JACCARD &&
metric_type != knowhere::Metric::TANIMOTO) {
metric_type != knowhere::Metric::JACCARD) {
std::string msg = "Index metric type " + metric_type +
" does not match index type " + index_type;
LOG_SERVER_ERROR_ << msg;
@ -398,8 +397,7 @@ ValidateSearchMetricType(const std::string_view metric_type, bool is_binary) {
} else {
// float
if (metric_type == knowhere::Metric::HAMMING ||
metric_type == knowhere::Metric::JACCARD ||
metric_type == knowhere::Metric::TANIMOTO) {
metric_type == knowhere::Metric::JACCARD) {
std::string msg =
"Cannot search float entities with index metric type " +
metric_type;

2
internal/core/unittest/test_index_wrapper.cpp
View File

@ -123,8 +123,6 @@ INSTANTIATE_TEST_CASE_P(
knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
knowhere::metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2)));

2
internal/core/unittest/test_indexing.cpp
View File

@ -360,8 +360,6 @@ INSTANTIATE_TEST_CASE_P(
knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
knowhere::metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::metric::JACCARD),
#ifdef BUILD_DISK_ANN

1
internal/core/unittest/test_range_search_sort.cpp
View File

@ -162,7 +162,6 @@ INSTANTIATE_TEST_CASE_P(RangeSearchSortParameters,
::testing::Values(knowhere::metric::L2,
knowhere::metric::IP,
knowhere::metric::JACCARD,
knowhere::metric::TANIMOTO,
knowhere::metric::HAMMING));
TEST_P(RangeSearchSortTest, CheckRangeSearchSort) {

4
internal/core/unittest/test_similarity_corelation.cpp
View File

@ -15,11 +15,9 @@
TEST(SimilarityCorelation, Naive) {
ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::IP));
ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::COSINE));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::L2));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::HAMMING));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::JACCARD));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::TANIMOTO));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUBSTRUCTURE));
ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUPERSTRUCTURE));
}

5
internal/proxy/util.go
View File

@ -43,6 +43,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/crypto"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -439,11 +440,11 @@ func isVector(dataType schemapb.DataType) (bool, error) {
func validateMetricType(dataType schemapb.DataType, metricTypeStrRaw string) error {
metricTypeStr := strings.ToUpper(metricTypeStrRaw)
switch metricTypeStr {
case "L2", "IP":
case metric.L2, metric.IP, metric.COSINE:
if dataType == schemapb.DataType_FloatVector {
return nil
}
case "JACCARD", "HAMMING", "TANIMOTO", "SUBSTRUCTURE", "SUBPERSTURCTURE":
case metric.JACCARD, metric.HAMMING:
if dataType == schemapb.DataType_BinaryVector {
return nil
}

3
internal/querynodev2/delegator/delegator_data_test.go
View File

@ -39,6 +39,7 @@ import (
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -117,7 +118,7 @@ func (s *DelegatorDataSuite) SetupTest() {
},
{
Key: common.MetricTypeKey,
Value: "TANIMOTO",
Value: metric.JACCARD,
},
},
},

3
internal/querynodev2/delegator/delegator_test.go
View File

@ -42,6 +42,7 @@ import (
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -139,7 +140,7 @@ func (s *DelegatorSuite) SetupTest() {
},
{
Key: common.MetricTypeKey,
Value: "TANIMOTO",
Value: metric.JACCARD,
},
},
},

11
internal/querynodev2/segments/mock_data.go
View File

@ -46,6 +46,7 @@ import (
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -59,12 +60,6 @@ const (
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"
IndexHNSW = "HNSW"
L2 = "L2"
IP = "IP"
hamming = "HAMMING"
Jaccard = "JACCARD"
tanimoto = "TANIMOTO"
nlist = 100
m = 4
nbits = 8
@ -79,7 +74,7 @@ const (
timestampFieldID = 1
metricTypeKey = common.MetricTypeKey
defaultDim = 128
defaultMetricType = "L2"
defaultMetricType = metric.L2
dimKey = common.DimKey
@ -113,7 +108,7 @@ var simpleFloatVecField = vecFieldParam{
var simpleBinVecField = vecFieldParam{
id: 101,
dim: defaultDim,
metricType: Jaccard,
metricType: metric.JACCARD,
vecType: schemapb.DataType_BinaryVector,
fieldName: "binVectorField",
}

5
internal/querynodev2/segments/segment_loader_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -234,7 +235,7 @@ func (suite *SegmentLoaderSuite) TestLoadWithIndex() {
vecFields[0],
msgLength,
IndexFaissIVFFlat,
L2,
metric.L2,
suite.chunkManager,
)
suite.NoError(err)
@ -401,7 +402,7 @@ func (suite *SegmentLoaderSuite) TestPatchEntryNum() {
vecFields[0],
msgLength,
IndexFaissIVFFlat,
L2,
metric.L2,
suite.chunkManager,
)
suite.NoError(err)

3
internal/querynodev2/segments/segment_test.go
View File

@ -12,6 +12,7 @@ import (
storage "github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -187,7 +188,7 @@ func (suite *SegmentSuite) TestValidateIndexedFieldsData() {
// with index but doesn't have raw data
index := suite.sealed.GetIndex(101)
_, indexParams := genIndexParams(IndexHNSW, L2)
_, indexParams := genIndexParams(IndexHNSW, metric.L2)
index.IndexInfo.IndexParams = funcutil.Map2KeyValuePair(indexParams)
DeleteSegment(suite.sealed)
suite.True(suite.sealed.ExistIndex(101))

3
internal/querynodev2/services_test.go
View File

@ -45,6 +45,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
@ -478,7 +479,7 @@ func (suite *ServiceSuite) genSegmentLoadInfos(schema *schemapb.CollectionSchema
vecFieldIDs[0],
1000,
segments.IndexFaissIVFFlat,
segments.L2,
metric.L2,
suite.node.vectorStorage,
)
suite.Require().NoError(err)

7
internal/util/indexcgowrapper/codec_index_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/metric"
)
type indexTestCase struct {
@ -201,7 +202,7 @@ func genFloatVecIndexCases(dtype schemapb.DataType) []indexTestCase {
typeParams: nil,
indexParams: map[string]string{
common.IndexTypeKey: IndexFaissIVFPQ,
common.MetricTypeKey: L2,
common.MetricTypeKey: metric.L2,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"m": strconv.Itoa(m),
@ -213,7 +214,7 @@ func genFloatVecIndexCases(dtype schemapb.DataType) []indexTestCase {
typeParams: nil,
indexParams: map[string]string{
common.IndexTypeKey: IndexFaissIVFFlat,
common.MetricTypeKey: L2,
common.MetricTypeKey: metric.L2,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
},
@ -228,7 +229,7 @@ func genBinaryVecIndexCases(dtype schemapb.DataType) []indexTestCase {
typeParams: nil,
indexParams: map[string]string{
common.IndexTypeKey: IndexFaissBinIVFFlat,
common.MetricTypeKey: Jaccard,
common.MetricTypeKey: metric.JACCARD,
common.DimKey: strconv.Itoa(dim),
"nlist": strconv.Itoa(nlist),
"nbits": strconv.Itoa(nbits),

37
internal/util/indexcgowrapper/index_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -22,13 +23,6 @@ const (
IndexHNSW = "HNSW"
// metric type
L2 = "L2"
IP = "IP"
hamming = "HAMMING"
Jaccard = "JACCARD"
tanimoto = "TANIMOTO"
dim = 8
nlist = 100
m = 4
@ -50,26 +44,25 @@ type vecTestCase struct {
func generateFloatVectorTestCases() []vecTestCase {
return []vecTestCase{
{IndexFaissIDMap, L2, false, schemapb.DataType_FloatVector},
{IndexFaissIDMap, IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFFlat, L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFFlat, IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFPQ, L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFPQ, IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, IP, false, schemapb.DataType_FloatVector},
{IndexHNSW, L2, false, schemapb.DataType_FloatVector},
{IndexHNSW, IP, false, schemapb.DataType_FloatVector},
{IndexFaissIDMap, metric.L2, false, schemapb.DataType_FloatVector},
{IndexFaissIDMap, metric.IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFFlat, metric.L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFFlat, metric.IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFPQ, metric.L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFPQ, metric.IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, metric.L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, metric.IP, false, schemapb.DataType_FloatVector},
{IndexHNSW, metric.L2, false, schemapb.DataType_FloatVector},
{IndexHNSW, metric.IP, false, schemapb.DataType_FloatVector},
}
}
func generateBinaryVectorTestCases() []vecTestCase {
return []vecTestCase{
{IndexFaissBinIVFFlat, Jaccard, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIVFFlat, hamming, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIVFFlat, tanimoto, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIDMap, Jaccard, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIDMap, hamming, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIVFFlat, metric.JACCARD, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIVFFlat, metric.HAMMING, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIDMap, metric.JACCARD, true, schemapb.DataType_BinaryVector},
{IndexFaissBinIDMap, metric.HAMMING, true, schemapb.DataType_BinaryVector},
}
}

16
pkg/util/indexparamcheck/bin_flat_checker_test.go
View File

@ -31,7 +31,6 @@ func Test_binFlatChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.HAMMING,
@ -40,18 +39,6 @@ func Test_binFlatChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -64,9 +51,6 @@ func Test_binFlatChecker_CheckTrain(t *testing.T) {
{p3, false},
{p4, true},
{p5, true},
{p6, true},
{p7, true},
{p8, true},
}
c := newBinFlatChecker()

4
pkg/util/indexparamcheck/bin_ivf_flat_checker.go
View File

@ -9,8 +9,8 @@ type binIVFFlatChecker struct {
}
func (c binIVFFlatChecker) StaticCheck(params map[string]string) error {
if !CheckStrByValues(params, Metric, BinIvfMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", BinIvfMetrics)
if !CheckStrByValues(params, Metric, BinMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", BinMetrics)
}
if !CheckIntByRange(params, NLIST, MinNList, MaxNList) {

27
pkg/util/indexparamcheck/bin_ivf_flat_checker_test.go
View File

@ -65,7 +65,6 @@ func Test_binIVFFlatChecker_CheckTrain(t *testing.T) {
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.HAMMING,
@ -80,28 +79,6 @@ func Test_binIVFFlatChecker_CheckTrain(t *testing.T) {
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.TANIMOTO,
NLIST: strconv.Itoa(100),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUBSTRUCTURE,
NLIST: strconv.Itoa(100),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUPERSTRUCTURE,
NLIST: strconv.Itoa(100),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
}
cases := []struct {
params map[string]string
@ -119,10 +96,6 @@ func Test_binIVFFlatChecker_CheckTrain(t *testing.T) {
{p4, true},
{p5, true},
{p6, true},
{p7, false},
{p8, false},
}
c := newBinIVFFlatChecker()

4
pkg/util/indexparamcheck/binary_vector_base_checker.go
View File

@ -13,8 +13,8 @@ type binaryVectorBaseChecker struct {
}
func (c binaryVectorBaseChecker) staticCheck(params map[string]string) error {
if !CheckStrByValues(params, Metric, BinIDMapMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", BinIDMapMetrics)
if !CheckStrByValues(params, Metric, BinMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", BinMetrics)
}
return nil

4
pkg/util/indexparamcheck/constraints.go
View File

@ -47,9 +47,7 @@ const (
var METRICS = []string{metric.L2, metric.IP, metric.COSINE} // const
// BinIDMapMetrics is a set of all metric types supported for binary vector.
var BinIDMapMetrics = []string{metric.HAMMING, metric.JACCARD, metric.TANIMOTO, metric.SUBSTRUCTURE,
metric.SUPERSTRUCTURE} // const
var BinIvfMetrics = []string{metric.HAMMING, metric.JACCARD, metric.TANIMOTO} // const
var BinMetrics = []string{metric.HAMMING, metric.JACCARD} // const
var HnswMetrics = []string{metric.L2, metric.IP, metric.COSINE, metric.HAMMING, metric.JACCARD} // const
var supportDimPerSubQuantizer = []int{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1} // const
var supportSubQuantizer = []int{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1} // const

16
pkg/util/indexparamcheck/diskann_checker_test.go
View File

@ -37,7 +37,6 @@ func Test_diskannChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.HAMMING,
@ -46,18 +45,6 @@ func Test_diskannChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -72,9 +59,6 @@ func Test_diskannChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newDiskannChecker()

17
pkg/util/indexparamcheck/flat_checker_test.go
View File

@ -23,7 +23,6 @@ func Test_flatChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.HAMMING,
@ -32,18 +31,7 @@ func Test_flatChecker_CheckTrain(t *testing.T) {
DIM: strconv.Itoa(128),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
errIsNil bool
@ -53,9 +41,6 @@ func Test_flatChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newFlatChecker()

22
pkg/util/indexparamcheck/hnsw_checker_test.go
View File

@ -49,7 +49,6 @@ func Test_hnswChecker_CheckTrain(t *testing.T) {
EFConstruction: strconv.Itoa(200),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
HNSWM: strconv.Itoa(16),
@ -62,24 +61,6 @@ func Test_hnswChecker_CheckTrain(t *testing.T) {
EFConstruction: strconv.Itoa(200),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
HNSWM: strconv.Itoa(16),
EFConstruction: strconv.Itoa(200),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
HNSWM: strconv.Itoa(16),
EFConstruction: strconv.Itoa(200),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
HNSWM: strconv.Itoa(16),
EFConstruction: strconv.Itoa(200),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -95,9 +76,6 @@ func Test_hnswChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, true},
{p5, true},
{p6, false},
{p7, false},
{p8, false},
}
c := newHnswChecker()

19
pkg/util/indexparamcheck/ivf_base_checker_test.go
View File

@ -32,7 +32,6 @@ func Test_ivfBaseChecker_CheckTrain(t *testing.T) {
NLIST: strconv.Itoa(1024),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
@ -43,21 +42,6 @@ func Test_ivfBaseChecker_CheckTrain(t *testing.T) {
NLIST: strconv.Itoa(1024),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -71,9 +55,6 @@ func Test_ivfBaseChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newIVFBaseChecker()

25
pkg/util/indexparamcheck/ivf_pq_checker_test.go
View File

@ -84,7 +84,6 @@ func Test_ivfPQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
@ -99,27 +98,6 @@ func Test_ivfPQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -142,9 +120,6 @@ func Test_ivfPQChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newIVFPQChecker()

22
pkg/util/indexparamcheck/ivf_sq_checker_test.go
View File

@ -46,7 +46,6 @@ func Test_ivfSQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(100),
@ -59,24 +58,6 @@ func Test_ivfSQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(100),
NBITS: strconv.Itoa(8),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(100),
NBITS: strconv.Itoa(8),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(100),
NBITS: strconv.Itoa(8),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -92,9 +73,6 @@ func Test_ivfSQChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newIVFSQChecker()

25
pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go
View File

@ -77,7 +77,6 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.COSINE,
}
p4 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
@ -92,27 +91,6 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
NBITS: strconv.Itoa(8),
Metric: metric.JACCARD,
}
p6 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.TANIMOTO,
}
p7 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.SUBSTRUCTURE,
}
p8 := map[string]string{
DIM: strconv.Itoa(128),
NLIST: strconv.Itoa(1024),
IVFM: strconv.Itoa(4),
NBITS: strconv.Itoa(8),
Metric: metric.SUPERSTRUCTURE,
}
cases := []struct {
params map[string]string
@ -134,9 +112,6 @@ func Test_raftIVFPQChecker_CheckTrain(t *testing.T) {
{p3, true},
{p4, false},
{p5, false},
{p6, false},
{p7, false},
{p8, false},
}
c := newRaftIVFPQChecker()

9
pkg/util/metric/metric_type.go
View File

@ -30,13 +30,4 @@ const (
// JACCARD represents jaccard distance
JACCARD MetricType = "JACCARD"
// TANIMOTO represents tanimoto distance
TANIMOTO MetricType = "TANIMOTO"
// SUBSTRUCTURE represents substructure distance
SUBSTRUCTURE MetricType = "SUBSTRUCTURE"
// SUPERSTRUCTURE represents superstructure distance
SUPERSTRUCTURE MetricType = "SUPERSTRUCTURE"
)

14
pkg/util/metric/similarity_corelation_test.go
View File

@ -28,12 +28,8 @@ func TestPositivelyRelated(t *testing.T) {
true,
},
{
JACCARD,
false,
},
{
TANIMOTO,
false,
COSINE,
true,
},
{
L2,
@ -44,11 +40,7 @@ func TestPositivelyRelated(t *testing.T) {
false,
},
{
SUPERSTRUCTURE,
false,
},
{
SUBSTRUCTURE,
JACCARD,
false,
},
}

2
tests/python_client/common/common_type.py
View File

@ -237,7 +237,7 @@ delete_support = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]
ivf = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]
skip_pq = ["IVF_PQ"]
float_metrics = ["L2", "IP", "COSINE"]
binary_metrics = ["JACCARD", "HAMMING", "TANIMOTO", "SUBSTRUCTURE", "SUPERSTRUCTURE"]
binary_metrics = ["JACCARD", "HAMMING"]
structure_metrics = ["SUBSTRUCTURE", "SUPERSTRUCTURE"]
all_scalar_data_types = ['int8', 'int16', 'int32', 'int64', 'float', 'double', 'bool', 'varchar']

2
tests/python_client/utils/util_pymilvus.py
View File

@ -75,7 +75,7 @@ def skip_pq():
def binary_metrics():
return ["JACCARD", "HAMMING", "TANIMOTO", "SUBSTRUCTURE", "SUPERSTRUCTURE"]
return ["JACCARD", "HAMMING"]
def structure_metrics():