milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change default shard number to 1 (#23586) 

Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>
pull/23595/head
zhenshan.cao 2023-04-21 07:08:32 +08:00 committed by GitHub
parent 66cd15f72b
commit 03ec804e68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 43 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/datanode/mock_test.go
View File

@ -1050,7 +1050,7 @@ func (m *RootCoordFactory) DescribeCollectionInternal(ctx context.Context, in *m
resp.CollectionID = m.collectionID
resp.Schema = meta.Schema
resp.ShardsNum = 2
resp.ShardsNum = common.DefaultShardsNum
resp.Status.ErrorCode = commonpb.ErrorCode_Success
return resp, nil
}

14
internal/metastore/db/dao/collection_test.go
View File

@ -240,7 +240,7 @@ func TestCollection_Get(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,
@ -266,7 +266,7 @@ func TestCollection_Get_Error(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,
@ -290,7 +290,7 @@ func TestCollection_Get_ErrRecordNotFound(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,
@ -358,7 +358,7 @@ func TestCollection_Insert(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Properties: propertiesStr,
@ -387,7 +387,7 @@ func TestCollection_Insert_Error(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,
@ -444,7 +444,7 @@ func Test_collectionDb_Update(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,
@ -472,7 +472,7 @@ func Test_collectionDb_Update(t *testing.T) {
CollectionName: "test_collection_name_1",
Description: "",
AutoID: false,
ShardsNum: int32(2),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Ts: ts,

4
internal/metastore/db/dbmodel/collection_test.go
View File

@ -42,7 +42,7 @@ func TestUnmarshalCollectionModel(t *testing.T) {
CollectionName: "cn",
Description: "",
AutoID: false,
ShardsNum: int32(1),
ShardsNum: common.DefaultShardsNum,
StartPosition: "",
ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually),
Properties: "",
@ -61,7 +61,7 @@ func TestUnmarshalCollectionModel(t *testing.T) {
assert.Equal(t, "cn", ret.Name)
assert.Equal(t, "", ret.Description)
assert.Equal(t, false, ret.AutoID)
assert.Equal(t, int32(1), ret.ShardsNum)
assert.Equal(t, common.DefaultShardsNum, ret.ShardsNum)
assert.Equal(t, 0, len(ret.StartPositions))
assert.Equal(t, commonpb.ConsistencyLevel(3), ret.ConsistencyLevel)
assert.Nil(t, ret.Properties)

8
internal/metastore/model/collection_test.go
View File

@ -3,11 +3,11 @@ package model
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/stretchr/testify/assert"
)
var (
@ -40,7 +40,7 @@ var (
Fields: []*Field{fieldModel},
VirtualChannelNames: []string{"vch"},
PhysicalChannelNames: []string{"pch"},
ShardsNum: 1,
ShardsNum: common.DefaultShardsNum,
CreateTime: 1,
StartPositions: startPositions,
ConsistencyLevel: commonpb.ConsistencyLevel_Strong,
@ -79,7 +79,7 @@ var (
},
VirtualChannelNames: []string{"vch"},
PhysicalChannelNames: []string{"pch"},
ShardsNum: 1,
ShardsNum: common.DefaultShardsNum,
StartPositions: startPositions,
ConsistencyLevel: commonpb.ConsistencyLevel_Strong,
Properties: []*commonpb.KeyValuePair{

2
internal/proxy/proxy_test.go
View File

@ -517,7 +517,7 @@ func TestProxy(t *testing.T) {
otherCollectionName := collectionName + "_other_" + funcutil.GenRandomStr()
partitionName := partitionPrefix + funcutil.GenRandomStr()
otherPartitionName := partitionPrefix + "_other_" + funcutil.GenRandomStr()
shardsNum := int32(2)
shardsNum := common.DefaultShardsNum
int64Field := "int64"
floatVecField := "fVec"
dim := 128

2
internal/proxy/task_query_test.go
View File

@ -36,7 +36,7 @@ func TestQueryTask_all(t *testing.T) {
qc = types.NewMockQueryCoord(t)
qn = &QueryNodeMock{}
shardsNum = int32(2)
shardsNum = common.DefaultShardsNum
collectionName = t.Name() + funcutil.GenRandomStr()
expr = fmt.Sprintf("%s > 0", testInt64Field)

6
internal/proxy/task_search_test.go
View File

@ -28,10 +28,6 @@ import (
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
const (
testShardsNum = int32(2)
)
func TestSearchTask_PostExecute(t *testing.T) {
t.Run("Test empty result", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
@ -73,7 +69,7 @@ func createColl(t *testing.T, name string, rc types.RootCoord) {
CreateCollectionRequest: &milvuspb.CreateCollectionRequest{
CollectionName: name,
Schema: marshaledSchema,
ShardsNum: testShardsNum,
ShardsNum: common.DefaultShardsNum,
},
ctx: ctx,
rootCoord: rc,

3
internal/proxy/task_statistic_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
@ -32,7 +33,7 @@ func TestStatisticTask_all(t *testing.T) {
qc = types.NewMockQueryCoord(t)
qn = types.NewMockQueryNode(t)
shardsNum = int32(2)
shardsNum = common.DefaultShardsNum
collectionName = t.Name() + funcutil.GenRandomStr()
)

4
internal/proxy/task_test.go
View File

@ -476,7 +476,7 @@ func TestCreateCollectionTask(t *testing.T) {
rc.Start()
defer rc.Stop()
ctx := context.Background()
shardsNum := int32(2)
shardsNum := common.DefaultShardsNum
prefix := "TestCreateCollectionTask"
dbName := ""
collectionName := prefix + funcutil.GenRandomStr()
@ -911,7 +911,7 @@ func TestDescribeCollectionTask_ShardsNum1(t *testing.T) {
dbName := ""
collectionName := prefix + funcutil.GenRandomStr()
shardsNum := int32(2)
shardsNum := common.DefaultShardsNum
int64Field := "int64"
floatVecField := "fvec"
dim := 128

3
internal/rootcoord/create_collection_task.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/metastore/model"
pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
ms "github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
@ -133,7 +134,7 @@ func (t *createCollectionTask) prepareSchema() error {
func (t *createCollectionTask) assignShardsNum() {
if t.Req.GetShardsNum() <= 0 {
t.Req.ShardsNum = 2
t.Req.ShardsNum = common.DefaultShardsNum
}
}

3
internal/rootcoord/create_collection_task_test.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@ -289,7 +290,7 @@ func Test_createCollectionTask_Prepare(t *testing.T) {
task.Req.ShardsNum = int32(Params.RootCoordCfg.DmlChannelNum.GetAsInt() + 1) // no enough channels.
err = task.Prepare(context.Background())
assert.Error(t, err)
task.Req.ShardsNum = 1
task.Req.ShardsNum = common.DefaultShardsNum
err = task.Prepare(context.Background())
assert.NoError(t, err)
})

11
internal/rootcoord/garbage_collector_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus/internal/metastore/model"
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
mocktso "github.com/milvus-io/milvus/internal/tso/mocks"
"github.com/milvus-io/milvus/pkg/common"
)
func TestGarbageCollectorCtx_ReDropCollection(t *testing.T) {
@ -113,8 +114,8 @@ func TestGarbageCollectorCtx_ReDropCollection(t *testing.T) {
core.ddlTsLockManager = newDdlTsLockManager(core.tsoAllocator)
gc := newBgGarbageCollector(core)
core.garbageCollector = gc
shardsNum := 2
pchans := ticker.getDmlChannelNames(shardsNum)
shardsNum := common.DefaultShardsNum
pchans := ticker.getDmlChannelNames(int(shardsNum))
gc.ReDropCollection(&model.Collection{PhysicalChannelNames: pchans}, 1000)
<-releaseCollectionChan
assert.True(t, releaseCollectionCalled)
@ -333,7 +334,7 @@ func TestGarbageCollectorCtx_RemoveCreatingCollection(t *testing.T) {
func TestGarbageCollectorCtx_ReDropPartition(t *testing.T) {
t.Run("failed to GcPartitionData", func(t *testing.T) {
ticker := newTickerWithMockFailStream() // failed to broadcast drop msg.
shardsNum := 2
shardsNum := int(common.DefaultShardsNum)
pchans := ticker.getDmlChannelNames(shardsNum)
tsoAllocator := newMockTsoAllocator()
tsoAllocator.GenerateTSOF = func(count uint32) (uint64, error) {
@ -348,7 +349,7 @@ func TestGarbageCollectorCtx_ReDropPartition(t *testing.T) {
t.Run("failed to RemovePartition", func(t *testing.T) {
ticker := newTickerWithMockNormalStream()
shardsNum := 2
shardsNum := int(common.DefaultShardsNum)
pchans := ticker.getDmlChannelNames(shardsNum)
meta := newMockMetaTable()
meta.RemovePartitionFunc = func(ctx context.Context, collectionID UniqueID, partitionID UniqueID, ts Timestamp) error {
@ -367,7 +368,7 @@ func TestGarbageCollectorCtx_ReDropPartition(t *testing.T) {
t.Run("normal case", func(t *testing.T) {
ticker := newTickerWithMockNormalStream()
shardsNum := 2
shardsNum := int(common.DefaultShardsNum)
pchans := ticker.getDmlChannelNames(shardsNum)
meta := newMockMetaTable()
removePartitionCalled := false

2
pkg/common/common.go
View File

@ -42,7 +42,7 @@ const (
TimeStampFieldName = "Timestamp"
// DefaultShardsNum defines the default number of shards when creating a collection
DefaultShardsNum = int32(2)
DefaultShardsNum = int32(1)
// InvalidPartitionID indicates that the partition is not specified. It will be set when the partitionName is empty
InvalidPartitionID = int64(-1)

8
scripts/run_go_unittest.sh
View File

@ -64,14 +64,14 @@ done
function test_proxy()
{
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast -count=1
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast -count=1 -v
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1 -v
}
function test_querynode()
{
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynode/..." -failfast -count=1
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynodev2/..." -failfast -count=1
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynodev2/..." -failfast -count=1
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1
}

2
tests/integration/bulkinsert_test.go
View File

@ -124,7 +124,7 @@ func TestBulkInsert(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)
if createCollectionStatus.GetErrorCode() != commonpb.ErrorCode_Success {

2
tests/integration/hello_milvus_test.go
View File

@ -102,7 +102,7 @@ func TestHelloMilvus(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)
if createCollectionStatus.GetErrorCode() != commonpb.ErrorCode_Success {

4
tests/integration/meta_watcher_test.go
View File

@ -115,7 +115,7 @@ func TestShowSegments(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)
assert.Equal(t, createCollectionStatus.GetErrorCode(), commonpb.ErrorCode_Success)
@ -207,7 +207,7 @@ func TestShowReplicas(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)
if createCollectionStatus.GetErrorCode() != commonpb.ErrorCode_Success {

4
tests/integration/range_search_test.go
View File

@ -99,7 +99,7 @@ func TestRangeSearchIP(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)
@ -337,7 +337,7 @@ func TestRangeSearchL2(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)

2
tests/integration/upsert_test.go
View File

@ -98,7 +98,7 @@ func TestUpsert(t *testing.T) {
DbName: dbName,
CollectionName: collectionName,
Schema: marshaledSchema,
ShardsNum: 2,
ShardsNum: common.DefaultShardsNum,
})
assert.NoError(t, err)