From 142848fcc3d289e47a239c54571c08e269941036 Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Fri, 18 Feb 2022 18:39:50 +0800 Subject: [PATCH] Abandon using protobuf to pass binaryset parameter (#15626) Signed-off-by: zhenshan.cao --- internal/core/CMakeLists.txt | 4 + internal/core/src/common/CMakeLists.txt | 5 +- internal/core/src/common/vector_index_c.cpp | 106 +++++++++++++++++ internal/core/src/common/vector_index_c.h | 52 +++++++++ internal/core/src/indexbuilder/CMakeLists.txt | 4 - .../core/src/indexbuilder/IndexWrapper.cpp | 38 ++++++- internal/core/src/indexbuilder/IndexWrapper.h | 7 ++ internal/core/src/indexbuilder/index_c.cpp | 36 ++++++ internal/core/src/indexbuilder/index_c.h | 7 ++ internal/core/src/segcore/CMakeLists.txt | 6 +- internal/core/src/segcore/load_index_c.cpp | 44 ------- internal/core/src/segcore/load_index_c.h | 11 +- internal/core/unittest/test_index_wrapper.cpp | 28 ++--- internal/indexnode/index.go | 107 +++++++++++++++++- internal/querynode/load_index_info.go | 10 +- 15 files changed, 374 insertions(+), 91 deletions(-) create mode 100644 internal/core/src/common/vector_index_c.cpp create mode 100644 internal/core/src/common/vector_index_c.h diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index 05a45e0db4..05699d749c 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -285,6 +285,10 @@ install(FILES ${CMAKE_BINARY_DIR}/src/indexbuilder/libmilvus_indexbuilder${CMAKE DESTINATION lib ) +install(FILES ${CMAKE_BINARY_DIR}/src/common/libmilvus_common${CMAKE_SHARED_LIBRARY_SUFFIX} + DESTINATION lib +) + # Install common install( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/common/ diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 04d23d8938..3ea17c1e89 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -13,10 +13,11 @@ set(COMMON_SRC Schema.cpp Types.cpp SystemProperty.cpp + vector_index_c.cpp ) -add_library(milvus_common +add_library(milvus_common SHARED ${COMMON_SRC} ) -target_link_libraries(milvus_common knowhere milvus_proto yaml-cpp ) +target_link_libraries(milvus_common milvus_utils milvus_config log knowhere milvus_proto yaml-cpp ) diff --git a/internal/core/src/common/vector_index_c.cpp b/internal/core/src/common/vector_index_c.cpp new file mode 100644 index 0000000000..9efd13913b --- /dev/null +++ b/internal/core/src/common/vector_index_c.cpp @@ -0,0 +1,106 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "knowhere/common/BinarySet.h" +#include "common/vector_index_c.h" + +CStatus +NewBinarySet(CBinarySet* c_binary_set) { + try { + auto binary_set = std::make_unique(); + *c_binary_set = binary_set.release(); + auto status = CStatus(); + status.error_code = Success; + status.error_msg = ""; + return status; + } catch (std::exception& e) { + auto status = CStatus(); + status.error_code = UnexpectedError; + status.error_msg = strdup(e.what()); + return status; + } +} + +void +DeleteBinarySet(CBinarySet c_binary_set) { + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + delete binary_set; +} + +CStatus +AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) { + auto status = CStatus(); + try { + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + std::string index_key(c_index_key); + uint8_t* index = (uint8_t*)index_binary; + std::shared_ptr data(index, [](void*) {}); + binary_set->Append(index_key, data, index_size); + + status.error_code = Success; + status.error_msg = ""; + } catch (std::exception& e) { + status.error_code = UnexpectedError; + status.error_msg = strdup(e.what()); + } + return status; +} + +int +GetBinarySetSize(CBinarySet c_binary_set) { + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + return binary_set->binary_map_.size(); +} + +void +GetBinarySetKeys(CBinarySet c_binary_set, void* datas) { + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto& map_ = binary_set->binary_map_; + const char** datas_ = (const char**)datas; + std::size_t i = 0; + for (auto it = map_.begin(); it != map_.end(); ++it, i++) { + datas_[i] = it->first.c_str(); + } +} + +int +GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) { + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + int64_t ret_ = 0; + try { + std::string key_(key); + auto binary = binary_set->GetByName(key_); + ret_ = binary->size; + } catch (std::exception& e) { + } + return ret_; +} + +CStatus +CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) { + auto status = CStatus(); + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + try { + auto binary = binary_set->GetByName(key); + status.error_code = Success; + status.error_msg = ""; + memcpy((uint8_t*)data, binary->data.get(), binary->size); + } catch (std::exception& e) { + status.error_code = UnexpectedError; + status.error_msg = strdup(e.what()); + } + return status; +} diff --git a/internal/core/src/common/vector_index_c.h b/internal/core/src/common/vector_index_c.h new file mode 100644 index 0000000000..3adad5dc48 --- /dev/null +++ b/internal/core/src/common/vector_index_c.h @@ -0,0 +1,52 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include "common/type_c.h" + +typedef void* CBinarySet; + +CStatus +NewBinarySet(CBinarySet* c_binary_set); + +void +DeleteBinarySet(CBinarySet c_binary_set); + +CStatus +AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key); + +int +GetBinarySetSize(CBinarySet c_binary_set); + +void +GetBinarySetKeys(CBinarySet c_binary_set, void* datas); + +int +GetBinarySetValueSize(CBinarySet c_set, const char* key); + +// Note: the memory of data has been allocated outside +CStatus +CopyBinarySetValue(void* data, const char* key, CBinarySet c_set); + +#ifdef __cplusplus +} +#endif diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt index f1ed3eca7e..4711989e9c 100644 --- a/internal/core/src/indexbuilder/CMakeLists.txt +++ b/internal/core/src/indexbuilder/CMakeLists.txt @@ -28,12 +28,8 @@ endif () # link order matters target_link_libraries(milvus_indexbuilder knowhere - milvus_config milvus_common - milvus_utils - milvus_proto ${TBB} - log ${PLATFORM_LIBS} pthread ) diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp index 169db0377c..55a0db2a10 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.cpp +++ b/internal/core/src/indexbuilder/IndexWrapper.cpp @@ -40,7 +40,8 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria AssertInfo(index_ != nullptr, "[IndexWrapper]Index is null after create index"); } -template // ugly here, ParamsT will just be MapParams later +template +// ugly here, ParamsT will just be MapParams later void IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Config& conf) { bool deserialized_success; @@ -164,7 +165,7 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { } } auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); - std::cout << "config_ when build index: " << config_ << std::endl; + std::cout << "Konwhere BuildWithoutIds config_ is " << config_ << std::endl; AssertInfo(conf_adapter->CheckTrain(config_, index_mode), "something wrong in index parameters!"); if (is_in_need_id_list(index_type)) { @@ -228,6 +229,23 @@ IndexWrapper::StoreRawData(const knowhere::DatasetPtr& dataset) { } } +std::unique_ptr +IndexWrapper::SerializeBinarySet() { + auto ret = std::make_unique(index_->Serialize(config_)); + auto index_type = get_index_type(); + + if (is_in_nm_list(index_type)) { + std::shared_ptr raw_data(new uint8_t[raw_data_.size()], std::default_delete()); + memcpy(raw_data.get(), raw_data_.data(), raw_data_.size()); + ret->Append(RAW_DATA, raw_data, raw_data_.size()); + auto slice_size = get_index_file_slice_size(); + // https://github.com/milvus-io/milvus/issues/6421 + // Disassemble will only divide the raw vectors, other keys were already divided + knowhere::Disassemble(slice_size * 1024 * 1024, *ret); + } + return std::move(ret); +} + /* * brief Return serialized binary set * TODO: use a more efficient method to manage memory, consider std::vector later @@ -236,6 +254,7 @@ std::unique_ptr IndexWrapper::Serialize() { auto binarySet = index_->Serialize(config_); auto index_type = get_index_type(); + if (is_in_nm_list(index_type)) { std::shared_ptr raw_data(new uint8_t[raw_data_.size()], std::default_delete()); memcpy(raw_data.get(), raw_data_.data(), raw_data_.size()); @@ -266,6 +285,21 @@ IndexWrapper::Serialize() { return binary; } +void +IndexWrapper::LoadFromBinarySet(milvus::knowhere::BinarySet& binary_set) { + auto& map_ = binary_set.binary_map_; + for (auto it = map_.begin(); it != map_.end(); ++it) { + if (it->first == RAW_DATA) { + raw_data_.clear(); + auto data_size = it->second->size; + raw_data_.resize(data_size); + memcpy(raw_data_.data(), it->second->data.get(), data_size); + break; + } + } + index_->Load(binary_set); +} + void IndexWrapper::Load(const char* serialized_sliced_blob_buffer, int32_t size) { namespace indexcgo = milvus::proto::indexcgo; diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h index 4d04842997..bb1f285d60 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.h +++ b/internal/core/src/indexbuilder/IndexWrapper.h @@ -17,6 +17,7 @@ #include #include "knowhere/index/vector_index/VecIndex.h" +#include "knowhere/common/BinarySet.h" namespace milvus::indexbuilder { @@ -37,6 +38,12 @@ class IndexWrapper { std::unique_ptr Serialize(); + std::unique_ptr + SerializeBinarySet(); + + void + LoadFromBinarySet(milvus::knowhere::BinarySet&); + void Load(const char* serialized_sliced_blob_buffer, int32_t size); diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index 65f85bf470..ab3ce135ba 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -10,9 +10,13 @@ // or implied. See the License for the specific language governing permissions and limitations under the License #include + #ifndef __APPLE__ + #include + #endif + #include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "indexbuilder/IndexWrapper.h" #include "indexbuilder/index_c.h" @@ -90,6 +94,22 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve return status; } +CStatus +SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set) { + auto status = CStatus(); + try { + auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto binary = cIndex->SerializeBinarySet(); + *c_binary_set = binary.release(); + status.error_code = Success; + status.error_msg = ""; + } catch (std::exception& e) { + status.error_code = UnexpectedError; + status.error_msg = strdup(e.what()); + } + return status; +} + CStatus SerializeToSlicedBuffer(CIndex index, CBinary* c_binary) { auto status = CStatus(); @@ -140,6 +160,22 @@ LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, in return status; } +CStatus +LoadFromBinarySet(CIndex index, CBinarySet c_binary_set) { + auto status = CStatus(); + try { + auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + cIndex->LoadFromBinarySet(*binary_set); + status.error_code = Success; + status.error_msg = ""; + } catch (std::exception& e) { + status.error_code = UnexpectedError; + status.error_msg = strdup(e.what()); + } + return status; +} + CStatus QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res) { auto status = CStatus(); diff --git a/internal/core/src/indexbuilder/index_c.h b/internal/core/src/indexbuilder/index_c.h index fe0c07bdf4..2a312f0034 100644 --- a/internal/core/src/indexbuilder/index_c.h +++ b/internal/core/src/indexbuilder/index_c.h @@ -18,6 +18,7 @@ extern "C" { #include #include "segcore/collection_c.h" #include "common/type_c.h" +#include "common/vector_index_c.h" typedef void* CIndex; typedef void* CIndexQueryResult; @@ -41,6 +42,9 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve CStatus SerializeToSlicedBuffer(CIndex index, CBinary* c_binary); +CStatus +SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set); + int64_t GetCBinarySize(CBinary c_binary); @@ -54,6 +58,9 @@ DeleteCBinary(CBinary c_binary); CStatus LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size); +CStatus +LoadFromBinarySet(CIndex index, CBinarySet c_binary_set); + CStatus QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res); diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 9d2fcc415d..02ca9228a8 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -40,16 +40,12 @@ endif () target_link_libraries(milvus_segcore ${PLATFORM_LIBS} - log pthread ${TBB} ${OpenMP_CXX_FLAGS} - knowhere milvus_common - milvus_config - milvus_proto + knowhere milvus_query - milvus_utils # gperftools ) diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp index c9a9195ad8..bdca33ca27 100644 --- a/internal/core/src/segcore/load_index_c.cpp +++ b/internal/core/src/segcore/load_index_c.cpp @@ -106,47 +106,3 @@ AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { return status; } } - -CStatus -NewBinarySet(CBinarySet* c_binary_set) { - try { - auto binary_set = std::make_unique(); - *c_binary_set = binary_set.release(); - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -void -DeleteBinarySet(CBinarySet c_binary_set) { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; - delete binary_set; -} - -CStatus -AppendBinaryIndex(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) { - try { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; - std::string index_key(c_index_key); - uint8_t* index = (uint8_t*)index_binary; - std::shared_ptr data(index, [](void*) {}); - binary_set->Append(index_key, data, index_size); - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} diff --git a/internal/core/src/segcore/load_index_c.h b/internal/core/src/segcore/load_index_c.h index 2e86f06f37..f8ae3a923a 100644 --- a/internal/core/src/segcore/load_index_c.h +++ b/internal/core/src/segcore/load_index_c.h @@ -17,11 +17,11 @@ extern "C" { #include #include +#include "common/vector_index_c.h" #include "common/type_c.h" #include "segcore/collection_c.h" typedef void* CLoadIndexInfo; -typedef void* CBinarySet; CStatus NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info); @@ -38,15 +38,6 @@ AppendFieldInfo(CLoadIndexInfo c_load_index_info, int64_t field_id); CStatus AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set); -CStatus -NewBinarySet(CBinarySet* c_binary_set); - -void -DeleteBinarySet(CBinarySet c_binary_set); - -CStatus -AppendBinaryIndex(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key); - #ifdef __cplusplus } #endif diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index e0ce33dcbc..4a8c2de029 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -315,16 +315,16 @@ TEST(IVFFLATNMWrapper, Codec) { auto dataset = GenDataset(flat_nb, metric_type, false); auto xb_data = dataset.get_col(0); auto xb_dataset = milvus::knowhere::GenDataset(flat_nb, DIM, xb_data.data()); - auto index = + auto index_wrapper = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); + ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset)); - auto binary = index->Serialize(); - auto copy_index = + auto binary = index_wrapper->Serialize(); + auto copy_index_wrapper = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(copy_index->Load(binary->data.data(), binary->data.size())); - ASSERT_EQ(copy_index->dim(), copy_index->dim()); - auto copy_binary = copy_index->Serialize(); + ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size())); + ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim()); + auto copy_binary = copy_index_wrapper->Serialize(); } TEST(BinFlatWrapper, Build) { @@ -416,17 +416,17 @@ TEST_P(IndexWrapperTest, BuildWithoutIds) { } TEST_P(IndexWrapperTest, Codec) { - auto index = + auto index_wrapper = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); + ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset)); - auto binary = index->Serialize(); - auto copy_index = + auto binary = index_wrapper->Serialize(); + auto copy_index_wrapper = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(copy_index->Load(binary->data.data(), binary->data.size())); - ASSERT_EQ(copy_index->dim(), copy_index->dim()); - auto copy_binary = copy_index->Serialize(); + ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size())); + ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim()); + auto copy_binary = copy_index_wrapper->Serialize(); if (!milvus::indexbuilder::is_in_nm_list(index_type)) { // binary may be not same due to uncertain internal map order ASSERT_EQ(binary->data.size(), copy_binary->data.size()); diff --git a/internal/indexnode/index.go b/internal/indexnode/index.go index 070f3be63a..ecfc90af86 100644 --- a/internal/indexnode/index.go +++ b/internal/indexnode/index.go @@ -20,12 +20,13 @@ package indexnode #cgo CFLAGS: -I${SRCDIR}/../core/output/include -#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath,"${SRCDIR}/../core/output/lib" -#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib +#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath,"${SRCDIR}/../core/output/lib" +#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib #include // free #include "segcore/collection_c.h" #include "indexbuilder/index_c.h" +#include "common/vector_index_c.h" */ import "C" @@ -33,6 +34,7 @@ import "C" import ( "errors" "fmt" + "path/filepath" "runtime" "unsafe" @@ -63,7 +65,75 @@ type CIndex struct { close bool } +func GetBinarySetKeys(cBinarySet C.CBinarySet) ([]string, error) { + size := int(C.GetBinarySetSize(cBinarySet)) + if size == 0 { + return nil, fmt.Errorf("BinarySet size is zero!") + } + datas := make([]unsafe.Pointer, size) + + C.GetBinarySetKeys(cBinarySet, unsafe.Pointer(&datas[0])) + ret := make([]string, size) + for i := 0; i < size; i++ { + ret[i] = C.GoString((*C.char)(datas[i])) + } + + return ret, nil +} + +func GetBinarySetValue(cBinarySet C.CBinarySet, key string) ([]byte, error) { + cIndexKey := C.CString(key) + defer C.free(unsafe.Pointer(cIndexKey)) + ret := C.GetBinarySetValueSize(cBinarySet, cIndexKey) + size := int(ret) + if size == 0 { + return nil, fmt.Errorf("GetBinarySetValueSize size is zero!") + } + value := make([]byte, size) + status := C.CopyBinarySetValue(unsafe.Pointer(&value[0]), cIndexKey, cBinarySet) + + if err := HandleCStatus(&status, "CopyBinarySetValue failed"); err != nil { + return nil, err + } + + return value, nil +} + +func (index *CIndex) Serialize() ([]*Blob, error) { + var cBinarySet C.CBinarySet + + status := C.SerializeToBinarySet(index.indexPtr, &cBinarySet) + defer func() { + if cBinarySet != nil { + C.DeleteBinarySet(cBinarySet) + } + }() + if err := HandleCStatus(&status, "SerializeToBinarySet failed"); err != nil { + return nil, err + } + + keys, err := GetBinarySetKeys(cBinarySet) + if err != nil { + return nil, err + } + ret := make([]*Blob, 0) + for _, key := range keys { + value, err := GetBinarySetValue(cBinarySet, key) + if err != nil { + return nil, err + } + blob := &Blob{ + Key: key, + Value: value, + } + ret = append(ret, blob) + } + + return ret, nil +} + // Serialize serializes vector data into bytes data so that it can be accessed in 'C'. +/* func (index *CIndex) Serialize() ([]*Blob, error) { var cBinary C.CBinary @@ -94,8 +164,36 @@ func (index *CIndex) Serialize() ([]*Blob, error) { return ret, nil } +*/ + +func (index *CIndex) Load(blobs []*Blob) error { + var cBinarySet C.CBinarySet + status := C.NewBinarySet(&cBinarySet) + defer C.DeleteBinarySet(cBinarySet) + + if err := HandleCStatus(&status, "CIndex Load2 NewBinarySet failed"); err != nil { + return err + } + for _, blob := range blobs { + key := blob.Key + byteIndex := blob.Value + indexPtr := unsafe.Pointer(&byteIndex[0]) + indexLen := C.int64_t(len(byteIndex)) + binarySetKey := filepath.Base(key) + log.Debug("", zap.String("index key", binarySetKey)) + indexKey := C.CString(binarySetKey) + status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey) + C.free(unsafe.Pointer(indexKey)) + if err := HandleCStatus(&status, "CIndex Load AppendIndexBinary failed"); err != nil { + return err + } + } + status = C.LoadFromBinarySet(index.indexPtr, cBinarySet) + return HandleCStatus(&status, "AppendIndex failed") +} // Load loads data from 'C'. +/* func (index *CIndex) Load(blobs []*Blob) error { binarySet := &indexcgopb.BinarySet{Datas: make([]*indexcgopb.Binary, 0)} for _, blob := range blobs { @@ -107,13 +205,10 @@ func (index *CIndex) Load(blobs []*Blob) error { return err2 } - /* - CStatus - LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size); - */ status := C.LoadFromSlicedBuffer(index.indexPtr, (*C.char)(unsafe.Pointer(&datas[0])), (C.int32_t)(len(datas))) return HandleCStatus(&status, "LoadFromSlicedBuffer failed") } +*/ // BuildFloatVecIndexWithoutIds builds indexes for float vector. func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error { diff --git a/internal/querynode/load_index_info.go b/internal/querynode/load_index_info.go index b413c94cac..8fe5dad049 100644 --- a/internal/querynode/load_index_info.go +++ b/internal/querynode/load_index_info.go @@ -18,13 +18,15 @@ package querynode /* #cgo CFLAGS: -I${SRCDIR}/../core/output/include -#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath,"${SRCDIR}/../core/output/lib" -#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib +#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_segcore -Wl,-rpath,"${SRCDIR}/../core/output/lib" +#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "segcore/load_index_c.h" +#include "common/vector_index_c.h" */ import "C" + import ( "path/filepath" "unsafe" @@ -108,9 +110,9 @@ func (li *LoadIndexInfo) appendIndexData(bytesIndex [][]byte, indexKeys []string binarySetKey := filepath.Base(indexKeys[i]) log.Debug("", zap.String("index key", binarySetKey)) indexKey := C.CString(binarySetKey) - status = C.AppendBinaryIndex(cBinarySet, indexPtr, indexLen, indexKey) + status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey) C.free(unsafe.Pointer(indexKey)) - if err := HandleCStatus(&status, "AppendBinaryIndex failed"); err != nil { + if err := HandleCStatus(&status, "LoadIndexInfo AppendIndexBinary failed"); err != nil { return err } }