milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix bulkinsert progress bug (#21680) 

Signed-off-by: groot <yihua.mo@zilliz.com>
pull/21712/head
groot 2023-01-13 16:25:40 +08:00 committed by GitHub
parent 90d9e165d4
commit c0e7a122d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/util/importutil/numpy_parser.go
View File

@ -387,6 +387,7 @@ func (p *NumpyParser) consume(columnReaders []*NumpyColumnReader) error {
tr := timerecord.NewTimeRecorder("consume performance") tr := timerecord.NewTimeRecorder("consume performance")
defer tr.Elapse("end") defer tr.Elapse("end")
// read data from files, batch by batch // read data from files, batch by batch
totalRead := 0
for { for {
readRowCount := 0 readRowCount := 0
segmentData := make(map[storage.FieldID]storage.FieldData) segmentData := make(map[storage.FieldID]storage.FieldData)
@ -411,7 +412,8 @@ func (p *NumpyParser) consume(columnReaders []*NumpyColumnReader) error {
if readRowCount == 0 { if readRowCount == 0 {
break break
} }
updateProgress(readRowCount) totalRead += readRowCount
updateProgress(totalRead)
tr.Record("readData") tr.Record("readData")
// split data to shards // split data to shards
err = p.splitFieldsData(segmentData, shards) err = p.splitFieldsData(segmentData, shards)