milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Support varchar autoid for bulkinsertV1 (#30896) 

This PR is a supplement to PR
https://github.com/milvus-io/milvus/pull/30377.

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
pull/30372/head
yihao.dai 2024-02-28 21:20:59 +08:00 committed by GitHub
parent 095cdbed59
commit 3775464b7c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
internal/util/importutil/json_handler.go
View File

@ -181,17 +181,17 @@ func (v *JSONRowConsumer) Handle(rows []map[storage.FieldID]interface{}) error {
var shard uint32
var partitionID int64
if primaryValidator.isString {
var pk string
if primaryValidator.autoID {
log.Warn("JSON row consumer: string type primary key cannot be auto-generated")
return merr.WrapErrImportFailed("string type primary key cannot be auto-generated")
}
value := row[primaryKeyID]
pk, err := getKeyValue(value, primaryValidator.fieldName, primaryValidator.isString)
if err != nil {
log.Warn("JSON row consumer: failed to parse primary key at the row",
zap.Int64("rowNumber", rowNumber), zap.Error(err))
return merr.WrapErrImportFailed(fmt.Sprintf("failed to parse primary key at the row %d, error: %v", rowNumber, err))
pk = strconv.FormatInt(rowIDBegin+int64(i), 10)
} else {
value := row[primaryKeyID]
pk, err = getKeyValue(value, primaryValidator.fieldName, primaryValidator.isString)
if err != nil {
log.Warn("JSON row consumer: failed to parse primary key at the row",
zap.Int64("rowNumber", rowNumber), zap.Error(err))
return merr.WrapErrImportFailed(fmt.Sprintf("failed to parse primary key at the row %d, error: %v", rowNumber, err))
}
}
// hash to shard based on pk, hash to partition if partition key exist

13
internal/util/importutil/json_handler_test.go
View File

@ -457,17 +457,19 @@ func Test_JSONRowConsumerHandleVarcharPK(t *testing.T) {
consumer := createConsumeFunc(shardNum, []int64{partitionID}, flushFunc)
consumer.shardsData = createShardsData(schema, nil, shardNum, []int64{partitionID})
// string type primary key cannot be auto-generated
input := make([]map[storage.FieldID]interface{}, 1)
input[0] = map[int64]interface{}{
101: true,
102: json.Number("1"),
103: json.Number("1.56"),
}
consumer.collectionInfo.PrimaryKey.AutoID = true
consumer.validators[101].autoID = true
err := consumer.Handle(input)
assert.Error(t, err)
consumer.collectionInfo.PrimaryKey.AutoID = false
assert.NoError(t, err)
callTime--
flushedRowCount--
consumer.rowCounter = 0
consumer.validators[101].autoID = false
// failed to parse primary key
err = consumer.Handle(input)
@ -550,10 +552,9 @@ func Test_JSONRowConsumerHandleVarcharPK(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, intputRowCount, flushedRowCount)
// string type primary key cannot be auto-generated
consumer.validators[101].autoID = true
err = consumer.Handle(input)
assert.Error(t, err)
assert.NoError(t, err)
})
}