fix: [2.5] Fix 0 read count during import (#38695)

issue: https://github.com/milvus-io/milvus/issues/38693

pr: https://github.com/milvus-io/milvus/pull/38694

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
pull/38737/head
yihao.dai 2024-12-24 22:08:49 +08:00 committed by GitHub
parent 91130909a5
commit bca21bde30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 48 additions and 1 deletions

View File

@ -84,8 +84,15 @@ func EstimateReadCountPerBatch(bufferSize int, schema *schemapb.CollectionSchema
if err != nil {
return 0, err
}
if sizePerRecord <= 0 || bufferSize <= 0 {
return 0, fmt.Errorf("invalid size, sizePerRecord=%d, bufferSize=%d", sizePerRecord, bufferSize)
}
if 1000*sizePerRecord <= bufferSize {
return 1000, nil
}
return int64(bufferSize) / int64(sizePerRecord), nil
ret := int64(bufferSize) / int64(sizePerRecord)
if ret <= 0 {
return 1, nil
}
return ret, nil
}

View File

@ -66,3 +66,43 @@ func TestUtil_EstimateReadCountPerBatch(t *testing.T) {
_, err = EstimateReadCountPerBatch(16*1024*1024, schema)
assert.Error(t, err)
}
func TestUtil_EstimateReadCountPerBatch_InvalidBufferSize(t *testing.T) {
schema := &schemapb.CollectionSchema{}
count, err := EstimateReadCountPerBatch(16*1024*1024, schema)
assert.Error(t, err)
assert.Equal(t, int64(0), count)
t.Logf("err=%v", err)
schema = &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
FieldID: 100,
DataType: schemapb.DataType_Int64,
},
},
}
count, err = EstimateReadCountPerBatch(0, schema)
assert.Error(t, err)
assert.Equal(t, int64(0), count)
t.Logf("err=%v", err)
}
func TestUtil_EstimateReadCountPerBatch_LargeSchema(t *testing.T) {
schema := &schemapb.CollectionSchema{}
for i := 0; i < 100; i++ {
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
FieldID: int64(i),
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.MaxLengthKey,
Value: "10000000",
},
},
})
}
count, err := EstimateReadCountPerBatch(16*1024*1024, schema)
assert.NoError(t, err)
assert.Equal(t, int64(1), count)
}