milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: Folders are also considered for WalkWithObjects to adhere bulk insert failure on using GCS buckets (#39352) 

Related task: [#39134
](https://github.com/milvus-io/milvus/issues/39134)

Previous PR: [#39150 ](https://github.com/milvus-io/milvus/pull/39150)
pull/39420/head
Gifi Siby 2025-01-19 00:19:03 +05:30 committed by GitHub
parent 5b35fc700d
commit f54e8e9554
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/storage/gcp_native_object_storage.go
View File

@ -170,10 +170,13 @@ func (gcs *GcpNativeObjectStorage) WalkWithObjects(ctx context.Context,
return checkObjectStorageError(prefix, err)
}
if objAttrs.Prefix != "" {
continue
}
if !walkFunc(&ChunkObjectInfo{FilePath: objAttrs.Name, ModifyTime: objAttrs.Updated}) {
return nil
if !walkFunc(&ChunkObjectInfo{FilePath: objAttrs.Prefix, ModifyTime: objAttrs.Updated}) {
return nil
}
} else if objAttrs.Name != "" {
if !walkFunc(&ChunkObjectInfo{FilePath: objAttrs.Name, ModifyTime: objAttrs.Updated}) {
return nil
}
}
}
return nil

15
internal/storage/gcp_native_object_storage_test.go
View File

@ -147,14 +147,11 @@ func TestGcpNativeObjectStorage(t *testing.T) {
value []byte
}{
{false, "abc/", []byte("123")},
{true, "abc/d/", []byte("1234")},
{false, "abc/d/e", []byte("12345")},
{true, "abc/d", []byte("1234")},
{true, "abc/e/d", []byte("12354")},
{true, "key_/1/1", []byte("111")},
{true, "key_/1/2/", []byte("222")},
{false, "key_/1/2/3", []byte("333")},
{true, "key_/1/2", []byte("222")},
{true, "key_/2/3", []byte("333")},
{true, "key_/test.txt", []byte("333")},
}
for _, test := range prepareTests {
@ -172,10 +169,10 @@ func TestGcpNativeObjectStorage(t *testing.T) {
prefix string
expectedValue []string
}{
{true, "abc/", []string{"abc/e/d"}},
{true, "key_/", []string{"key_/1/1", "key_/2/3", "key_/test.txt"}},
{false, "abc/", []string{}},
{false, "key_/", []string{"key_/test.txt"}},
{true, "abc/", []string{"abc/d", "abc/e/d"}},
{true, "key_/", []string{"key_/1/1", "key_/1/2", "key_/2/3"}},
{false, "abc/", []string{"abc/d", "abc/e/"}},
{false, "key_/", []string{"key_/1/", "key_/2/"}},
}
for _, test := range insertWithPrefixTests {