mirror of https://github.com/milvus-io/milvus.git
127 lines
3.8 KiB
Go
127 lines
3.8 KiB
Go
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
|
|
package datacoord
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
|
"github.com/milvus-io/milvus/internal/proto/internalpb"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestFlushMonitor(t *testing.T) {
|
|
const collID = UniqueID(0)
|
|
const partID0 = UniqueID(100)
|
|
const partID1 = UniqueID(101)
|
|
const channelName = "c1"
|
|
|
|
mockAllocator := newMockAllocator()
|
|
meta, err := newMemoryMeta(mockAllocator)
|
|
assert.Nil(t, err)
|
|
|
|
testSchema := newTestSchema()
|
|
collInfo := &datapb.CollectionInfo{
|
|
ID: collID,
|
|
Schema: testSchema,
|
|
Partitions: []UniqueID{partID0, partID1},
|
|
}
|
|
|
|
meta.AddCollection(collInfo)
|
|
|
|
// create seg0 for partition0, seg0/seg1 for partition1
|
|
segID0_0, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
segInfo0_0, err := buildSegment(collID, partID0, segID0_0, channelName)
|
|
assert.Nil(t, err)
|
|
segID1_0, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
segInfo1_0, err := buildSegment(collID, partID1, segID1_0, channelName)
|
|
assert.Nil(t, err)
|
|
segID1_1, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
segInfo1_1, err := buildSegment(collID, partID1, segID1_1, channelName)
|
|
assert.Nil(t, err)
|
|
|
|
// check AddSegment
|
|
err = meta.AddSegment(segInfo0_0)
|
|
assert.Nil(t, err)
|
|
err = meta.AddSegment(segInfo0_0)
|
|
assert.NotNil(t, err)
|
|
err = meta.AddSegment(segInfo1_0)
|
|
assert.Nil(t, err)
|
|
err = meta.AddSegment(segInfo1_1)
|
|
assert.Nil(t, err)
|
|
|
|
t.Run("Test empty flush monitor", func(t *testing.T) {
|
|
fm := emptyFlushMonitor(meta)
|
|
ids := fm.CheckSegments([]*datapb.SegmentInfo{})
|
|
assert.Equal(t, 0, len(ids))
|
|
|
|
ids = fm.CheckChannels([]string{channelName}, nil)
|
|
assert.Equal(t, 0, len(ids))
|
|
})
|
|
|
|
t.Run("Test custom segment policy", func(t *testing.T) {
|
|
fm := emptyFlushMonitor(meta)
|
|
fm.segmentPolicy = estSegmentSizePolicy(1024*1024, 1024*1024*2) // row size 1Mib Limit 2 MB
|
|
segID3Rows, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
segInfo3Rows, err := buildSegment(collID, partID1, segID3Rows, channelName)
|
|
segInfo3Rows.NumOfRows = 3
|
|
assert.Nil(t, err)
|
|
|
|
ids := fm.CheckSegments([]*datapb.SegmentInfo{segInfo3Rows})
|
|
if assert.Equal(t, 1, len(ids)) {
|
|
assert.Equal(t, segID3Rows, ids[0])
|
|
}
|
|
})
|
|
|
|
t.Run("Test custom channel policy", func(t *testing.T) {
|
|
const channelName2 = `ch2`
|
|
fm := emptyFlushMonitor(meta)
|
|
fm.channelPolicy = channelSizeEpochPolicy(100, uint64(time.Hour))
|
|
|
|
for i := 0; i < 100; i++ {
|
|
segID, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
seg, err := buildSegment(collID, partID0, segID, channelName2)
|
|
assert.Nil(t, err)
|
|
seg.DmlPosition = &internalpb.MsgPosition{
|
|
Timestamp: uint64(i + 1),
|
|
}
|
|
meta.AddSegment(seg)
|
|
}
|
|
|
|
ids := fm.CheckChannels([]string{channelName2}, nil)
|
|
assert.Equal(t, 0, len(ids))
|
|
|
|
exSegID, err := mockAllocator.allocID()
|
|
assert.Nil(t, err)
|
|
seg, err := buildSegment(collID, partID0, exSegID, channelName2)
|
|
assert.Nil(t, err)
|
|
seg.DmlPosition = &internalpb.MsgPosition{
|
|
Timestamp: uint64(0), // the oldest
|
|
}
|
|
meta.AddSegment(seg)
|
|
|
|
ids = fm.CheckChannels([]string{channelName2}, nil)
|
|
if assert.Equal(t, 1, len(ids)) {
|
|
assert.Equal(t, exSegID, ids[0])
|
|
}
|
|
|
|
ids = fm.CheckChannels([]string{channelName2}, &internalpb.MsgPosition{Timestamp: uint64(time.Hour + 5)})
|
|
assert.Equal(t, 5, len(ids))
|
|
})
|
|
}
|