Improve codecov for paramtable and typeutil (#7451)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

internal/proxy/plan_parser.go

@@ -534,7 +534,7 @@ func (context *ParserContext) handleLeafValue(nodeRaw *ant_ast.Node, dataType sc
 					FloatVal: float64(node.Value),
 				},
 			}
-		} else if typeutil.IsIntergerType(dataType) {
+		} else if typeutil.IsIntegerType(dataType) {
 			gv = &planpb.GenericValue{
 				Val: &planpb.GenericValue_Int64Val{
 					Int64Val: int64(node.Value),

internal/proxy/plan_parser_test.go

@@ -32,7 +32,7 @@ func newTestSchema() *schemapb.CollectionSchema {
 
 	for name, value := range schemapb.DataType_value {
 		dataType := schemapb.DataType(value)
-		if !typeutil.IsIntergerType(dataType) && !typeutil.IsFloatingType(dataType) && !typeutil.IsVectorType(dataType) {
+		if !typeutil.IsIntegerType(dataType) && !typeutil.IsFloatingType(dataType) && !typeutil.IsVectorType(dataType) {
 			continue
 		}
 		newField := &schemapb.FieldSchema{

internal/storage/vector_chunk_manager_test.go

@@ -63,7 +63,7 @@ func TestVectorChunkManager(t *testing.T) {
 
 	floatResult := make([]float32, 0)
 	for i := 0; i < len(content)/4; i++ {
-		singleData := typeutil.ByteToFloat32(content[i*4 : i*4+4])
+		singleData := typeutil.BytesToFloat32(content[i*4 : i*4+4])
 		floatResult = append(floatResult, singleData)
 	}
 	assert.Equal(t, []float32{0, 1, 2, 3, 4, 5, 6, 7, 0, 111, 222, 333, 444, 555, 777, 666}, floatResult)
@@ -75,7 +75,7 @@ func TestVectorChunkManager(t *testing.T) {
 
 	floatResult = make([]float32, 0)
 	for i := 0; i < len(content)/4; i++ {
-		singleData := typeutil.ByteToFloat32(content[i*4 : i*4+4])
+		singleData := typeutil.BytesToFloat32(content[i*4 : i*4+4])
 		floatResult = append(floatResult, singleData)
 	}
 	assert.Equal(t, []float32{0, 111, 222, 333, 444, 555, 777, 666}, floatResult)
@@ -119,7 +119,7 @@ func TestVectorChunkManagerWithLocalCache(t *testing.T) {
 
 	floatResult := make([]float32, 0)
 	for i := 0; i < len(content)/4; i++ {
-		singleData := typeutil.ByteToFloat32(content[i*4 : i*4+4])
+		singleData := typeutil.BytesToFloat32(content[i*4 : i*4+4])
 		floatResult = append(floatResult, singleData)
 	}
 	assert.Equal(t, []float32{0, 1, 2, 3, 4, 5, 6, 7, 0, 111, 222, 333, 444, 555, 777, 666}, floatResult)
@@ -131,7 +131,7 @@ func TestVectorChunkManagerWithLocalCache(t *testing.T) {
 
 	floatResult = make([]float32, 0)
 	for i := 0; i < len(content)/4; i++ {
-		singleData := typeutil.ByteToFloat32(content[i*4 : i*4+4])
+		singleData := typeutil.BytesToFloat32(content[i*4 : i*4+4])
 		floatResult = append(floatResult, singleData)
 	}
 	assert.Equal(t, []float32{0, 111, 222, 333, 444, 555, 777, 666}, floatResult)

internal/util/paramtable/basetable_test.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/milvus-io/milvus/internal/proto/commonpb"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -26,9 +27,7 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
-//func TestMain
-
-func TestGlobalParamsTable_SaveAndLoad(t *testing.T) {
+func TestBaseParams_SaveAndLoad(t *testing.T) {
 	err1 := baseParams.Save("int", "10")
 	assert.Nil(t, err1)
 
@@ -57,7 +56,30 @@ func TestGlobalParamsTable_SaveAndLoad(t *testing.T) {
 	assert.Nil(t, err6)
 }
 
-func TestGlobalParamsTable_LoadRange(t *testing.T) {
+func TestBaseParams_LoadFromKVPair(t *testing.T) {
+	var kvPairs []*commonpb.KeyValuePair
+	kvPairs = append(kvPairs, &commonpb.KeyValuePair{
+		Key:   "k1",
+		Value: "v1",
+	})
+	kvPairs = append(kvPairs, &commonpb.KeyValuePair{
+		Key:   "k2",
+		Value: "v2",
+	})
+
+	err := baseParams.LoadFromKVPair(kvPairs)
+	assert.Nil(t, err)
+
+	v, err := baseParams.Load("k1")
+	assert.Nil(t, err)
+	assert.Equal(t, "v1", v)
+
+	v, err = baseParams.Load("k2")
+	assert.Nil(t, err)
+	assert.Equal(t, "v2", v)
+}
+
+func TestBaseParams_LoadRange(t *testing.T) {
 	_ = baseParams.Save("xxxaab", "10")
 	_ = baseParams.Save("xxxfghz", "20")
 	_ = baseParams.Save("xxxbcde", "1.1")
@@ -79,7 +101,7 @@ func TestGlobalParamsTable_LoadRange(t *testing.T) {
 	_ = baseParams.Remove("zhi")
 }
 
-func TestGlobalParamsTable_Remove(t *testing.T) {
+func TestBaseParams_Remove(t *testing.T) {
 	err1 := baseParams.Save("RemoveInt", "10")
 	assert.Nil(t, err1)
 
@@ -99,15 +121,76 @@ func TestGlobalParamsTable_Remove(t *testing.T) {
 	assert.Nil(t, err6)
 }
 
-func TestGlobalParamsTable_LoadYaml(t *testing.T) {
+func TestBaseParams_LoadYaml(t *testing.T) {
 	err := baseParams.LoadYaml("milvus.yaml")
 	assert.Nil(t, err)
-
 	err = baseParams.LoadYaml("advanced/channel.yaml")
 	assert.Nil(t, err)
+	assert.Panics(t, func() { baseParams.LoadYaml("advanced/not_exist.yaml") })
 
 	_, err = baseParams.Load("etcd.address")
 	assert.Nil(t, err)
 	_, err = baseParams.Load("pulsar.port")
 	assert.Nil(t, err)
 }
+
+func TestBaseParams_Parse(t *testing.T) {
+	t.Run("ParseBool", func(t *testing.T) {
+		assert.Nil(t, baseParams.Save("key", "true"))
+		assert.True(t, baseParams.ParseBool("key", false))
+		assert.False(t, baseParams.ParseBool("not_exist_key", false))
+
+		assert.Nil(t, baseParams.Save("key", "rand"))
+		assert.Panics(t, func() { baseParams.ParseBool("key", false) })
+	})
+
+	t.Run("ParseFloat", func(t *testing.T) {
+		assert.Nil(t, baseParams.Save("key", "0"))
+		assert.Equal(t, float64(0), baseParams.ParseFloat("key"))
+
+		assert.Nil(t, baseParams.Save("key", "3.14"))
+		assert.Equal(t, float64(3.14), baseParams.ParseFloat("key"))
+
+		assert.Panics(t, func() { baseParams.ParseFloat("not_exist_key") })
+		assert.Nil(t, baseParams.Save("key", "abc"))
+		assert.Panics(t, func() { baseParams.ParseFloat("key") })
+	})
+
+	t.Run("ParseInt32", func(t *testing.T) {
+		assert.Nil(t, baseParams.Save("key", "0"))
+		assert.Equal(t, int32(0), baseParams.ParseInt32("key"))
+
+		assert.Nil(t, baseParams.Save("key", "314"))
+		assert.Equal(t, int32(314), baseParams.ParseInt32("key"))
+
+		assert.Panics(t, func() { baseParams.ParseInt32("not_exist_key") })
+		assert.Nil(t, baseParams.Save("key", "abc"))
+		assert.Panics(t, func() { baseParams.ParseInt32("key") })
+	})
+
+	t.Run("ParseInt64", func(t *testing.T) {
+		assert.Nil(t, baseParams.Save("key", "0"))
+		assert.Equal(t, int64(0), baseParams.ParseInt64("key"))
+
+		assert.Nil(t, baseParams.Save("key", "314"))
+		assert.Equal(t, int64(314), baseParams.ParseInt64("key"))
+
+		assert.Panics(t, func() { baseParams.ParseInt64("not_exist_key") })
+		assert.Nil(t, baseParams.Save("key", "abc"))
+		assert.Panics(t, func() { baseParams.ParseInt64("key") })
+	})
+}
+
+func TestBaseParams_ConvertRange(t *testing.T) {
+	t.Run("ConvertRangeToIntSlice", func(t *testing.T) {
+		slice := ConvertRangeToIntSlice("0,10", ",")
+		assert.Equal(t, 10, len(slice))
+
+		assert.Panics(t, func() { ConvertRangeToIntSlice("0", ",") })
+		assert.Panics(t, func() { ConvertRangeToIntSlice("0, 10", ",") })
+		assert.Panics(t, func() { ConvertRangeToIntSlice("abc,10", ",") })
+		assert.Panics(t, func() { ConvertRangeToIntSlice("0,abc", ",") })
+		assert.Panics(t, func() { ConvertRangeToIntSlice("-1,9", ",") })
+		assert.Panics(t, func() { ConvertRangeToIntSlice("9,0", ",") })
+	})
+}

internal/util/paramtable/param_test.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package paramtable
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParam(t *testing.T) {
+	Params.Init()
+
+	assert.NotZero(t, len(Params.EtcdEndpoints))
+	t.Logf("etcd endpoints = %s", Params.EtcdEndpoints)
+
+	assert.NotEqual(t, Params.MetaRootPath, "")
+	t.Logf("meta root path = %s", Params.MetaRootPath)
+
+	assert.NotEqual(t, Params.KvRootPath, "")
+	t.Logf("kv root path = %s", Params.KvRootPath)
+
+	// test UseEmbedEtcd
+	Params.Save("etcd.use.embed", "true")
+	assert.Nil(t, os.Setenv("DEPLOY_MODE", "DISTRIBUTED"))
+	assert.Panics(t, func() { Params.initUseEmbedEtcd() })
+
+	assert.Nil(t, os.Setenv("DEPLOY_MODE", "STANDALONE"))
+	Params.LoadCfgToMemory()
+}

internal/util/typeutil/convension.go

@@ -18,8 +18,8 @@ import (
 	"reflect"
 )
 
-// Float32ToByte converts a float to byte slice.
-func Float32ToByte(float float32) []byte {
+// Float32ToBytes converts a float to byte slice.
+func Float32ToBytes(float float32) []byte {
 	bits := math.Float32bits(float)
 	bytes := make([]byte, 4)
 	binary.LittleEndian.PutUint32(bytes, bits)
@@ -28,7 +28,7 @@ func Float32ToByte(float float32) []byte {
 }
 
 // BytesToFloat32 converts a byte slice to float32.
-func ByteToFloat32(bytes []byte) float32 {
+func BytesToFloat32(bytes []byte) float32 {
 	bits := binary.LittleEndian.Uint32(bytes)
 
 	return math.Float32frombits(bits)

internal/util/typeutil/conversion_test.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package typeutil
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConversion(t *testing.T) {
+	t.Run("TestConvertFloat32", func(t *testing.T) {
+		comp := func(f float32) {
+			fb := Float32ToBytes(f)
+			f1 := BytesToFloat32(fb)
+			assert.Less(t, math.Abs(float64(f)-float64(f1)), 0.00001)
+		}
+		comp(float32(3.14))
+		comp(float32(0))
+		comp(float32(-139.866))
+		comp(float32(math.MaxFloat32))
+		comp(float32(-math.MaxFloat32))
+	})
+
+	t.Run("TestConvertInt64", func(t *testing.T) {
+		comp := func(i int64) {
+			ib := Int64ToBytes(i)
+			i1, err := BytesToInt64(ib)
+			assert.Nil(t, err)
+			assert.Equal(t, i, i1)
+		}
+		comp(int64(314))
+		comp(int64(0))
+		comp(int64(-8654273))
+		comp(int64(math.MaxInt64))
+		comp(int64(math.MinInt64))
+	})
+
+	t.Run("TestConvertUint64", func(t *testing.T) {
+		comp := func(u uint64) {
+			ub := Uint64ToBytes(u)
+			u1, err := BytesToUint64(ub)
+			assert.Nil(t, err)
+			assert.Equal(t, u, u1)
+		}
+		comp(uint64(314))
+		comp(uint64(0))
+		comp(uint64(75123348654273))
+		comp(uint64(math.MaxUint64))
+	})
+
+	t.Run("TestSliceRemoveDuplicate", func(t *testing.T) {
+		ret := SliceRemoveDuplicate(1)
+		assert.Equal(t, 0, len(ret))
+
+		arr := []int64{1, 1, 1, 2, 2, 3}
+		ret1 := SliceRemoveDuplicate(arr)
+		assert.Equal(t, 3, len(ret1))
+	})
+}

internal/util/typeutil/hash_test.go

@@ -30,9 +30,13 @@ func TestHash32_Uint64(t *testing.T) {
 	var u uint64 = 0x12
 	h, err := Hash32Uint64(u)
 	assert.Nil(t, err)
-
 	t.Log(h)
 
+	h1, err := Hash32Int64(int64(u))
+	assert.Nil(t, err)
+	t.Log(h1)
+	assert.Equal(t, h, h1)
+
 	b := make([]byte, unsafe.Sizeof(u))
 	b[0] = 0x12
 	h2, err := Hash32Bytes(b)
@@ -56,5 +60,4 @@ func TestHash32_String(t *testing.T) {
 	log.Println(h2)
 
 	assert.Equal(t, uint32(h), h2)
-
 }

internal/util/typeutil/schema.go

@@ -143,7 +143,7 @@ func IsVectorType(dataType schemapb.DataType) bool {
 	}
 }
 
-func IsIntergerType(dataType schemapb.DataType) bool {
+func IsIntegerType(dataType schemapb.DataType) bool {
 	switch dataType {
 	case schemapb.DataType_Int8, schemapb.DataType_Int16,
 		schemapb.DataType_Int32, schemapb.DataType_Int64:

internal/util/typeutil/schema_test.go

@@ -0,0 +1,157 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package typeutil
+
+import (
+	"testing"
+
+	"github.com/milvus-io/milvus/internal/proto/commonpb"
+	"github.com/milvus-io/milvus/internal/proto/schemapb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSchema(t *testing.T) {
+	schema := &schemapb.CollectionSchema{
+		Name:        "testColl",
+		Description: "",
+		AutoID:      false,
+		Fields: []*schemapb.FieldSchema{
+			{
+				FieldID:      100,
+				Name:         "field_int8",
+				IsPrimaryKey: false,
+				Description:  "",
+				DataType:     2,
+			},
+			{
+				FieldID:      101,
+				Name:         "field_int16",
+				IsPrimaryKey: false,
+				Description:  "",
+				DataType:     3,
+			},
+			{
+				FieldID:      102,
+				Name:         "field_int32",
+				IsPrimaryKey: false,
+				Description:  "",
+				DataType:     4,
+			},
+			{
+				FieldID:      103,
+				Name:         "field_int64",
+				IsPrimaryKey: true,
+				Description:  "",
+				DataType:     5,
+			},
+			{
+				FieldID:      104,
+				Name:         "field_float_vector",
+				IsPrimaryKey: false,
+				Description:  "",
+				DataType:     101,
+				TypeParams: []*commonpb.KeyValuePair{
+					{
+						Key:   "dim",
+						Value: "128",
+					},
+				},
+				IndexParams: []*commonpb.KeyValuePair{
+					{
+						Key:   "nlist",
+						Value: "128",
+					},
+				},
+			},
+			{
+				FieldID:      105,
+				Name:         "field_binary_vector",
+				IsPrimaryKey: false,
+				Description:  "",
+				DataType:     100,
+				TypeParams: []*commonpb.KeyValuePair{
+					{
+						Key:   "dim",
+						Value: "128",
+					},
+				},
+			},
+		},
+	}
+
+	t.Run("EstimateSizePerRecord", func(t *testing.T) {
+		size, err := EstimateSizePerRecord(schema)
+		assert.Equal(t, 543, size)
+		assert.Nil(t, err)
+	})
+
+	t.Run("SchemaHelper", func(t *testing.T) {
+		helper, err := CreateSchemaHelper(schema)
+		assert.Nil(t, err)
+
+		field, err := helper.GetPrimaryKeyField()
+		assert.Nil(t, err)
+		assert.Equal(t, "field_int64", field.Name)
+
+		field1, err := helper.GetFieldFromName("field_int8")
+		assert.Nil(t, err)
+		assert.Equal(t, "field_int8", field1.Name)
+
+		field2, err := helper.GetFieldFromID(102)
+		assert.Nil(t, err)
+		assert.Equal(t, "field_int32", field2.Name)
+
+		dim, err := helper.GetVectorDimFromID(104)
+		assert.Nil(t, err)
+		assert.Equal(t, 128, dim)
+		dim1, err := helper.GetVectorDimFromID(105)
+		assert.Nil(t, err)
+		assert.Equal(t, 128, dim1)
+		_, err = helper.GetVectorDimFromID(103)
+		assert.NotNil(t, err)
+	})
+
+	t.Run("Type", func(t *testing.T) {
+		assert.False(t, IsVectorType(schemapb.DataType_Bool))
+		assert.False(t, IsVectorType(schemapb.DataType_Int8))
+		assert.False(t, IsVectorType(schemapb.DataType_Int16))
+		assert.False(t, IsVectorType(schemapb.DataType_Int32))
+		assert.False(t, IsVectorType(schemapb.DataType_Int64))
+		assert.False(t, IsVectorType(schemapb.DataType_Float))
+		assert.False(t, IsVectorType(schemapb.DataType_Double))
+		assert.False(t, IsVectorType(schemapb.DataType_String))
+		assert.True(t, IsVectorType(schemapb.DataType_BinaryVector))
+		assert.True(t, IsVectorType(schemapb.DataType_FloatVector))
+
+		assert.False(t, IsIntegerType(schemapb.DataType_Bool))
+		assert.True(t, IsIntegerType(schemapb.DataType_Int8))
+		assert.True(t, IsIntegerType(schemapb.DataType_Int16))
+		assert.True(t, IsIntegerType(schemapb.DataType_Int32))
+		assert.True(t, IsIntegerType(schemapb.DataType_Int64))
+		assert.False(t, IsIntegerType(schemapb.DataType_Float))
+		assert.False(t, IsIntegerType(schemapb.DataType_Double))
+		assert.False(t, IsIntegerType(schemapb.DataType_String))
+		assert.False(t, IsIntegerType(schemapb.DataType_BinaryVector))
+		assert.False(t, IsIntegerType(schemapb.DataType_FloatVector))
+
+		assert.False(t, IsFloatingType(schemapb.DataType_Bool))
+		assert.False(t, IsFloatingType(schemapb.DataType_Int8))
+		assert.False(t, IsFloatingType(schemapb.DataType_Int16))
+		assert.False(t, IsFloatingType(schemapb.DataType_Int32))
+		assert.False(t, IsFloatingType(schemapb.DataType_Int64))
+		assert.True(t, IsFloatingType(schemapb.DataType_Float))
+		assert.True(t, IsFloatingType(schemapb.DataType_Double))
+		assert.False(t, IsFloatingType(schemapb.DataType_String))
+		assert.False(t, IsFloatingType(schemapb.DataType_BinaryVector))
+		assert.False(t, IsFloatingType(schemapb.DataType_FloatVector))
+	})
+}

internal/util/typeutil/time_test.go

@@ -0,0 +1,33 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package typeutil
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseTimestamp(t *testing.T) {
+	ts, err := ParseTimestamp(Int64ToBytes(1000))
+	t.Log(ts.String())
+	assert.Nil(t, err)
+}
+
+func TestSubTimeByWallClock(t *testing.T) {
+	beg := time.Now()
+	time.Sleep(100 * time.Millisecond)
+	end := time.Now()
+	span := SubTimeByWallClock(end, beg)
+	t.Log(span.String())
+}