milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor duplicate error class into one place (#26985) 

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
pull/27151/head
Enwei Jiao 2023-09-11 20:43:17 +08:00 committed by GitHub
parent 1aa3a1e067
commit c3f15c6b95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
99 changed files with 303 additions and 2275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Makefile
View File

@ -68,7 +68,7 @@ getdeps:
echo "Installing mockery v$(MOCKERY_VERSION) to ./bin/" && GOBIN=$(INSTALL_PATH) go install github.com/vektra/mockery/v2@v$(MOCKERY_VERSION); \
else \
echo "Mockery v$(MOCKERY_VERSION) already installed"; \
fi
fi
tools/bin/revive: tools/check/go.mod
cd tools/check; \
@ -296,13 +296,10 @@ gpu-install: milvus-gpu
clean:
@echo "Cleaning up all the generated files"
@find . -name '*.test' | xargs rm -fv
@find . -name '*~' | xargs rm -fv
@rm -rf bin/
@rm -rf lib/
@rm -rf $(GOPATH)/bin/milvus
@rm -rf cmake_build
@rm -rf cwrapper_build
@rm -rf internal/core/output
milvus-tools: print-build-info
@ -336,9 +333,9 @@ generate-mockery-types: getdeps
# Proxy
$(INSTALL_PATH)/mockery --name=ProxyComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_proxy.go --with-expecter --structname=MockProxy
# QueryCoord
$(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord
$(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord
# QueryNode
$(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode
$(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode
# DataCoord
$(INSTALL_PATH)/mockery --name=DataCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_datacoord.go --with-expecter --structname=MockDataCoord
# DataNode
@ -384,9 +381,9 @@ generate-mockery-datanode: getdeps
$(INSTALL_PATH)/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator --outpkg=allocator --inpackage
generate-mockery-metastore: getdeps
$(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks
$(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks
$(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks
$(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks
$(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks
$(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks
generate-mockery-utils: getdeps
# dependency.Factory
@ -404,4 +401,4 @@ generate-mockery-pkg:
$(MAKE) -C pkg generate-mockery
generate-mockery: generate-mockery-types generate-mockery-kv generate-mockery-rootcoord generate-mockery-proxy generate-mockery-querycoord generate-mockery-querynode generate-mockery-datacoord generate-mockery-pkg

3
internal/core/src/CMakeLists.txt
View File

@ -23,10 +23,7 @@ endif()
include_directories(${MILVUS_ENGINE_SRC})
include_directories(${MILVUS_THIRDPARTY_SRC})
set(FOUND_OPENBLAS "unknown")
add_subdirectory( pb )
add_subdirectory( exceptions )
add_subdirectory( utils )
add_subdirectory( log )
add_subdirectory( config )
add_subdirectory( common )

2
internal/core/src/common/BitsetView.h
View File

@ -22,7 +22,7 @@
#include <deque>
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "knowhere/bitsetview.h"
namespace milvus {

37
internal/core/src/common/CGoHelper.h
View File

@ -1,37 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
#include <string.h>
#include "common/type_c.h"
namespace milvus {
inline CStatus
SuccessCStatus() {
return CStatus{Success, ""};
}
inline CStatus
FailureCStatus(ErrorCode error_code, const std::string_view str) {
auto str_dup = strdup(str.data());
return CStatus{error_code, str_dup};
}
} // namespace milvus

6
internal/core/src/common/CMakeLists.txt
View File

@ -20,14 +20,16 @@ set(COMMON_SRC
Common.cpp
RangeSearchHelper.cpp
Tracer.cpp
IndexMeta.cpp)
IndexMeta.cpp
EasyAssert.cpp
)
add_library(milvus_common SHARED ${COMMON_SRC})
target_link_libraries(milvus_common
milvus_config
milvus_utils
milvus_log
milvus_proto
yaml-cpp
boost_bitset_ext
simdjson

1
internal/core/src/exceptions/EasyAssert.cpp → internal/core/src/common/EasyAssert.cpp
View File

@ -16,7 +16,6 @@
#include <iostream>
#include "EasyAssert.h"
// #define BOOST_STACKTRACE_USE_BACKTRACE
#include <boost/stacktrace.hpp>
#include <sstream>

28
internal/core/src/exceptions/EasyAssert.h → internal/core/src/common/EasyAssert.h
View File

@ -22,10 +22,26 @@
#include <stdlib.h>
#include <string>
#include "pb/common.pb.h"
#include "common/type_c.h"
/* Paste this on the file if you want to debug. */
namespace milvus {
using ErrorCodeEnum = proto::common::ErrorCode;
enum ErrorCodeEnum {
Success = 0,
UnexpectedError = 1,
BuildIndexError = 2,
IllegalArgument = 5,
};
inline CStatus
SuccessCStatus() {
return CStatus{Success, ""};
}
inline CStatus
FailureCStatus(int code, const std::string& msg) {
return CStatus{code, strdup(msg.data())};
}
namespace impl {
void
EasyAssertInfo(bool value,
@ -39,6 +55,11 @@ EasyAssertInfo(bool value,
class SegcoreError : public std::runtime_error {
public:
static SegcoreError
success() {
return SegcoreError(ErrorCodeEnum::Success, "");
}
SegcoreError(ErrorCodeEnum error_code, const std::string& error_msg)
: std::runtime_error(error_msg), error_code_(error_code) {
}
@ -48,6 +69,11 @@ class SegcoreError : public std::runtime_error {
return error_code_;
}
bool
ok() {
return error_code_ == ErrorCodeEnum::Success;
}
private:
ErrorCodeEnum error_code_;
};

3
internal/core/src/common/FieldMeta.h
View File

@ -21,8 +21,7 @@
#include <string>
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "utils/Status.h"
#include "common/EasyAssert.h"
namespace milvus {

4
internal/core/src/utils/File.h → internal/core/src/common/File.h
View File

@ -12,7 +12,7 @@
#pragma once
#include <string>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "fmt/core.h"
#include <fcntl.h>
#include <unistd.h>
@ -62,4 +62,4 @@ class File {
}
int fd_{-1};
};
} // namespace milvus
} // namespace milvus

2
internal/core/src/common/Json.h
View File

@ -25,7 +25,7 @@
#include <string>
#include <string_view>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "simdjson.h"
#include "fmt/core.h"
#include "simdjson/common_defs.h"

2
internal/core/src/common/RangeSearchHelper.cpp
View File

@ -88,7 +88,7 @@ ReGenRangeSearchResult(DatasetPtr data_set,
pq(cmp);
auto capacity = std::min<int64_t>(lims[i + 1] - lims[i], topk);
for (int j = lims[i]; j < lims[i + 1]; j++) {
for (size_t j = lims[i]; j < lims[i + 1]; j++) {
auto curr = ResultPair(dist[j], id[j]);
if (pq.size() < capacity) {
pq.push(curr);

2
internal/core/src/common/SystemProperty.cpp
View File

@ -18,7 +18,7 @@
#include "SystemProperty.h"
#include "Consts.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
namespace milvus {
class SystemPropertyImpl : public SystemProperty {

1
internal/core/src/common/SystemProperty.h
View File

@ -19,7 +19,6 @@
#include <string>
#include "Types.h"
#include "utils/Json.h"
namespace milvus {

6
internal/core/src/common/Tracer.cpp
View File

@ -11,6 +11,8 @@
#include "log/Log.h"
#include "Tracer.h"
#include <utility>
#include "opentelemetry/exporters/ostream/span_exporter_factory.h"
#include "opentelemetry/exporters/jaeger/jaeger_exporter_factory.h"
#include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h"
@ -103,7 +105,7 @@ thread_local std::shared_ptr<trace::Span> local_span;
void
SetRootSpan(std::shared_ptr<trace::Span> span) {
if (enable_trace) {
local_span = span;
local_span = std::move(span);
}
}
@ -123,7 +125,7 @@ AddEvent(std::string event_label) {
bool
isEmptyID(const uint8_t* id, int length) {
for (size_t i = 0; i < length; i++) {
for (int i = 0; i < length; i++) {
if (id[i] != 0) {
return false;
}

2
internal/core/src/common/Utils.h
View File

@ -28,7 +28,7 @@
#include "common/FieldMeta.h"
#include "common/LoadInfo.h"
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "knowhere/dataset.h"
#include "knowhere/expected.h"
#include "simdjson.h"

13
internal/core/src/common/binary_set_c.cpp
View File

@ -14,6 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/EasyAssert.h"
#include "knowhere/binaryset.h"
#include "common/binary_set_c.h"
@ -23,12 +24,12 @@ NewBinarySet(CBinarySet* c_binary_set) {
auto binary_set = std::make_unique<knowhere::BinarySet>();
*c_binary_set = binary_set.release();
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::ErrorCodeEnum::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::ErrorCodeEnum::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -55,10 +56,10 @@ AppendIndexBinary(CBinarySet c_binary_set,
std::shared_ptr<uint8_t[]> data(dup);
binary_set->Append(index_key, data, index_size);
status.error_code = Success;
status.error_code = milvus::ErrorCodeEnum::Success;
status.error_msg = "";
} catch (std::exception& e) {
status.error_code = UnexpectedError;
status.error_code = milvus::ErrorCodeEnum::UnexpectedError;
status.error_msg = strdup(e.what());
}
return status;
@ -100,11 +101,11 @@ CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) {
auto binary_set = (knowhere::BinarySet*)c_binary_set;
try {
auto binary = binary_set->GetByName(key);
status.error_code = Success;
status.error_code = milvus::ErrorCodeEnum::Success;
status.error_msg = "";
memcpy((uint8_t*)data, binary->data.get(), binary->size);
} catch (std::exception& e) {
status.error_code = UnexpectedError;
status.error_code = milvus::ErrorCodeEnum::UnexpectedError;
status.error_msg = strdup(e.what());
}
return status;

4
internal/core/src/common/protobuf_utils.h
View File

@ -21,7 +21,7 @@
#include <google/protobuf/text_format.h>
#include "pb/schema.pb.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
using std::string;
@ -38,4 +38,4 @@ RepeatedKeyValToMap(
}
return mapping;
}
} //namespace milvus
} //namespace milvus

7
internal/core/src/common/type_c.h
View File

@ -31,12 +31,6 @@ enum SegmentType {
typedef enum SegmentType SegmentType;
enum ErrorCode {
Success = 0,
UnexpectedError = 1,
IllegalArgument = 5,
};
// pure C don't support that we use schemapb.DataType directly.
// Note: the value of all enumerations must match the corresponding schemapb.DataType.
// TODO: what if there are increments in schemapb.DataType.
@ -108,4 +102,5 @@ typedef struct CTraceContext {
} CTraceContext;
#ifdef __cplusplus
}
#endif

9
internal/core/src/config/CMakeLists.txt
View File

@ -20,13 +20,8 @@ if ( EMBEDDED_MILVUS )
add_compile_definitions( EMBEDDED_MILVUS )
endif()
set(CONFIG_SRC
ConfigKnowhere.cpp
)
set(CONFIG_SRC ConfigKnowhere.cpp)
add_library(milvus_config STATIC ${CONFIG_SRC})
target_link_libraries(milvus_config
milvus_exceptions
knowhere
)
target_link_libraries(milvus_config knowhere)

2
internal/core/src/config/ConfigKnowhere.cpp
View File

@ -17,7 +17,7 @@
#include <mutex>
#include "ConfigKnowhere.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "glog/logging.h"
#include "log/Log.h"
#include "knowhere/comp/thread_pool.h"

23
internal/core/src/exceptions/CMakeLists.txt
View File

@ -1,23 +0,0 @@
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set(exceptions_files
EasyAssert.cpp
)
add_library(milvus_exceptions STATIC ${exceptions_files})
target_link_libraries(milvus_exceptions milvus_proto)

13
internal/core/src/index/CMakeLists.txt
View File

@ -27,17 +27,6 @@ endif ()
milvus_add_pkg_config("milvus_index")
add_library(milvus_index SHARED ${INDEX_FILES})
set(PLATFORM_LIBS )
if ( LINUX OR APPLE )
set(PLATFORM_LIBS marisa)
endif()
if (MSYS)
set(PLATFORM_LIBS -Wl,--allow-multiple-definition)
endif ()
target_link_libraries(milvus_index
milvus_storage
${PLATFORM_LIBS}
)
target_link_libraries(milvus_index milvus_storage)
install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}")

2
internal/core/src/index/Index.h
View File

@ -18,7 +18,7 @@
#include <memory>
#include <boost/dynamic_bitset.hpp>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "knowhere/comp/index_param.h"
#include "knowhere/dataset.h"
#include "common/Types.h"

2
internal/core/src/index/ScalarIndex-inl.h
View File

@ -20,6 +20,7 @@
#include "index/Meta.h"
#include "knowhere/dataset.h"
#include "common/Types.h"
namespace milvus::index {
template <typename T>
@ -73,7 +74,6 @@ inline void
ScalarIndex<std::string>::BuildWithRawData(size_t n,
const void* values,
const Config& config) {
// TODO :: use arrow
proto::schema::StringArray arr;
auto ok = arr.ParseFromArray(values, n);
Assert(ok);

2
internal/core/src/index/ScalarIndex.h
View File

@ -22,7 +22,7 @@
#include <string>
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "index/Index.h"
namespace milvus::index {

8
internal/core/src/index/StringIndexMarisa.cpp
View File

@ -30,8 +30,6 @@
namespace milvus::index {
#if defined(__linux__) || defined(__APPLE__)
StringIndexMarisa::StringIndexMarisa(storage::FileManagerImplPtr file_manager) {
if (file_manager != nullptr) {
file_manager_ = std::dynamic_pointer_cast<storage::MemFileManagerImpl>(
@ -67,7 +65,7 @@ StringIndexMarisa::Build(const Config& config) {
marisa::Keyset keyset;
for (auto data : field_datas) {
auto slice_num = data->get_num_rows();
for (size_t i = 0; i < slice_num; ++i) {
for (int64_t i = 0; i < slice_num; ++i) {
keyset.push_back(
(*static_cast<const std::string*>(data->RawValue(i))).c_str());
}
@ -80,7 +78,7 @@ StringIndexMarisa::Build(const Config& config) {
int64_t offset = 0;
for (auto data : field_datas) {
auto slice_num = data->get_num_rows();
for (size_t i = 0; i < slice_num; ++i) {
for (int64_t i = 0; i < slice_num; ++i) {
auto str_id =
lookup(*static_cast<const std::string*>(data->RawValue(i)));
AssertInfo(valid_str_id(str_id), "invalid marisa key");
@ -390,6 +388,4 @@ StringIndexMarisa::Reverse_Lookup(size_t offset) const {
return std::string(agent.key().ptr(), agent.key().length());
}
#endif
} // namespace milvus::index

4
internal/core/src/index/StringIndexMarisa.h
View File

@ -16,8 +16,6 @@
#pragma once
#if defined(__linux__) || defined(__APPLE__)
#include <marisa.h>
#include "index/StringIndex.h"
#include <string>
@ -114,5 +112,3 @@ CreateStringIndexMarisa(storage::FileManagerImplPtr file_manager = nullptr) {
}
} // namespace milvus::index
#endif

4
internal/core/src/index/Utils.cpp
View File

@ -30,12 +30,12 @@
#include "index/Meta.h"
#include <google/protobuf/text_format.h>
#include <unistd.h>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "knowhere/comp/index_param.h"
#include "common/Slice.h"
#include "storage/FieldData.h"
#include "storage/Util.h"
#include "utils/File.h"
#include "common/File.h"
namespace milvus::index {

4
internal/core/src/index/VectorMemIndex.cpp
View File

@ -29,7 +29,7 @@
#include "index/Index.h"
#include "index/Meta.h"
#include "index/Utils.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "config/ConfigKnowhere.h"
#include "knowhere/factory.h"
#include "knowhere/comp/time_recorder.h"
@ -43,7 +43,7 @@
#include "storage/MemFileManagerImpl.h"
#include "storage/ThreadPools.h"
#include "storage/Util.h"
#include "utils/File.h"
#include "common/File.h"
#include "common/Tracer.h"
namespace milvus::index {

2
internal/core/src/indexbuilder/VecIndexCreator.cpp
View File

@ -11,7 +11,7 @@
#include <map>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "indexbuilder/VecIndexCreator.h"
#include "index/Utils.h"
#include "index/IndexFactory.h"

3
internal/core/src/indexbuilder/index_c.cpp
View File

@ -15,7 +15,7 @@
#include <malloc.h>
#endif
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "indexbuilder/VecIndexCreator.h"
#include "indexbuilder/index_c.h"
#include "indexbuilder/IndexFactory.h"
@ -26,6 +26,7 @@
#include "pb/index_cgo_msg.pb.h"
#include "storage/Util.h"
using namespace milvus;
CStatus
CreateIndex(enum CDataType dtype,
const char* serialized_type_params,

4
internal/core/src/mmap/Column.h
View File

@ -23,12 +23,12 @@
#include "common/FieldMeta.h"
#include "common/Span.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/File.h"
#include "fmt/format.h"
#include "log/Log.h"
#include "mmap/Utils.h"
#include "storage/FieldData.h"
#include "utils/File.h"
namespace milvus {

2
internal/core/src/mmap/Utils.h
View File

@ -27,7 +27,7 @@
#include "common/FieldMeta.h"
#include "mmap/Types.h"
#include "storage/Util.h"
#include "utils/File.h"
#include "common/File.h"
namespace milvus {

3
internal/core/src/query/CMakeLists.txt
View File

@ -10,7 +10,6 @@
# or implied. See the License for the specific language governing permissions and limitations under the License
set(MILVUS_QUERY_SRCS
deprecated/BinaryQuery.cpp
generated/PlanNode.cpp
generated/Expr.cpp
visitors/ShowPlanNodeVisitor.cpp
@ -32,6 +31,6 @@ set(MILVUS_QUERY_SRCS
add_library(milvus_query ${MILVUS_QUERY_SRCS})
if(USE_DYNAMIC_SIMD)
target_link_libraries(milvus_query milvus_index milvus_simd)
else()
else()
target_link_libraries(milvus_query milvus_index)
endif()

4
internal/core/src/query/PlanImpl.h
View File

@ -19,8 +19,8 @@
#include "Plan.h"
#include "PlanNode.h"
#include "exceptions/EasyAssert.h"
#include "utils/Json.h"
#include "common/EasyAssert.h"
#include "common/Json.h"
#include "common/Consts.h"
namespace milvus::query {

5
internal/core/src/query/PlanProto.cpp
View File

@ -18,7 +18,7 @@
#include "ExprImpl.h"
#include "common/VectorTrait.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "generated/ExtractInfoExprVisitor.h"
#include "generated/ExtractInfoPlanNodeVisitor.h"
#include "pb/plan.pb.h"
@ -182,7 +182,8 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
search_info.metric_type_ = query_info_proto.metric_type();
search_info.topk_ = query_info_proto.topk();
search_info.round_decimal_ = query_info_proto.round_decimal();
search_info.search_params_ = json::parse(query_info_proto.search_params());
search_info.search_params_ =
nlohmann::json::parse(query_info_proto.search_params());
auto plan_node = [&]() -> std::unique_ptr<VectorPlanNode> {
if (anns_proto.vector_type() ==

2
internal/core/src/query/Relational.h
View File

@ -16,7 +16,7 @@
#include "common/Utils.h"
#include "common/VectorTrait.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "query/Expr.h"
#include "query/Utils.h"

2
internal/core/src/query/SubSearchResult.cpp
View File

@ -11,7 +11,7 @@
#include <cmath>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "query/SubSearchResult.h"
namespace milvus::query {

324
internal/core/src/query/deprecated/BinaryQuery.cpp
View File

@ -1,324 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <memory>
#include <set>
#include <stack>
#include <string>
#include <utility>
#include <vector>
#include "BinaryQuery.h"
namespace milvus {
namespace query_old {
BinaryQueryPtr
ConstructBinTree(std::vector<BooleanQueryPtr> queries,
QueryRelation relation,
uint64_t idx) {
if (idx == queries.size()) {
return nullptr;
} else if (idx == queries.size() - 1) {
return queries[idx]->getBinaryQuery();
} else {
BinaryQueryPtr bquery = std::make_shared<BinaryQuery>();
bquery->relation = relation;
bquery->left_query = std::make_shared<GeneralQuery>();
bquery->right_query = std::make_shared<GeneralQuery>();
bquery->left_query->bin = queries[idx]->getBinaryQuery();
++idx;
bquery->right_query->bin = ConstructBinTree(queries, relation, idx);
return bquery;
}
}
Status
ConstructLeafBinTree(std::vector<LeafQueryPtr> leaf_queries,
BinaryQueryPtr binary_query,
uint64_t idx) {
if (idx == leaf_queries.size()) {
return Status::OK();
}
binary_query->left_query = std::make_shared<GeneralQuery>();
binary_query->right_query = std::make_shared<GeneralQuery>();
if (leaf_queries.size() == leaf_queries.size() - 1) {
binary_query->left_query->leaf = leaf_queries[idx];
return Status::OK();
} else if (idx == leaf_queries.size() - 2) {
binary_query->left_query->leaf = leaf_queries[idx];
++idx;
binary_query->right_query->leaf = leaf_queries[idx];
return Status::OK();
} else {
binary_query->left_query->bin->relation = binary_query->relation;
binary_query->right_query->leaf = leaf_queries[idx];
++idx;
return ConstructLeafBinTree(
leaf_queries, binary_query->left_query->bin, idx);
}
}
Status
GenBinaryQuery(BooleanQueryPtr query, BinaryQueryPtr& binary_query) {
if (query->getBooleanQueries().size() == 0) {
if (binary_query->relation == QueryRelation::AND ||
binary_query->relation == QueryRelation::OR) {
// Put VectorQuery to the end of leaf queries
auto query_size = query->getLeafQueries().size();
for (uint64_t i = 0; i < query_size; ++i) {
if (query->getLeafQueries()[i]->vector_placeholder.size() > 0) {
std::swap(query->getLeafQueries()[i],
query->getLeafQueries()[0]);
break;
}
}
return ConstructLeafBinTree(
query->getLeafQueries(), binary_query, 0);
} else {
switch (query->getOccur()) {
case Occur::MUST: {
binary_query->relation = QueryRelation::AND;
return GenBinaryQuery(query, binary_query);
}
case Occur::MUST_NOT:
binary_query->is_not = true;
case Occur::SHOULD: {
binary_query->relation = QueryRelation::OR;
return GenBinaryQuery(query, binary_query);
}
default:
return Status::OK();
}
}
}
if (query->getBooleanQueries().size() == 1) {
auto bc = query->getBooleanQueries()[0];
binary_query->left_query = std::make_shared<GeneralQuery>();
switch (bc->getOccur()) {
case Occur::MUST: {
binary_query->relation = QueryRelation::AND;
return GenBinaryQuery(bc, binary_query);
}
case Occur::MUST_NOT:
binary_query->is_not = true;
case Occur::SHOULD: {
binary_query->relation = QueryRelation::OR;
return GenBinaryQuery(bc, binary_query);
}
default:
return Status::OK();
}
}
// Construct binary query for every single boolean query
std::vector<BooleanQueryPtr> must_queries;
std::vector<BooleanQueryPtr> must_not_queries;
std::vector<BooleanQueryPtr> should_queries;
Status status;
for (auto& _query : query->getBooleanQueries()) {
status = GenBinaryQuery(_query, _query->getBinaryQuery());
if (!status.ok()) {
return status;
}
if (_query->getOccur() == Occur::MUST) {
must_queries.emplace_back(_query);
} else if (_query->getOccur() == Occur::MUST_NOT) {
must_not_queries.emplace_back(_query);
} else {
should_queries.emplace_back(_query);
}
}
// Construct binary query for combine boolean queries
BinaryQueryPtr must_bquery, should_bquery, must_not_bquery;
uint64_t bquery_num = 0;
if (must_queries.size() > 1) {
// Construct a must binary tree
must_bquery = ConstructBinTree(must_queries, QueryRelation::R1, 0);
++bquery_num;
} else if (must_queries.size() == 1) {
must_bquery = must_queries[0]->getBinaryQuery();
++bquery_num;
}
if (should_queries.size() > 1) {
// Construct a should binary tree
should_bquery = ConstructBinTree(should_queries, QueryRelation::R2, 0);
++bquery_num;
} else if (should_queries.size() == 1) {
should_bquery = should_queries[0]->getBinaryQuery();
++bquery_num;
}
if (must_not_queries.size() > 1) {
// Construct a must_not binary tree
must_not_bquery =
ConstructBinTree(must_not_queries, QueryRelation::R1, 0);
++bquery_num;
} else if (must_not_queries.size() == 1) {
must_not_bquery = must_not_queries[0]->getBinaryQuery();
++bquery_num;
}
binary_query->left_query = std::make_shared<GeneralQuery>();
binary_query->right_query = std::make_shared<GeneralQuery>();
BinaryQueryPtr must_should_query = std::make_shared<BinaryQuery>();
must_should_query->left_query = std::make_shared<GeneralQuery>();
must_should_query->right_query = std::make_shared<GeneralQuery>();
if (bquery_num == 3) {
must_should_query->relation = QueryRelation::R3;
must_should_query->left_query->bin = must_bquery;
must_should_query->right_query->bin = should_bquery;
binary_query->relation = QueryRelation::R1;
binary_query->left_query->bin = must_should_query;
binary_query->right_query->bin = must_not_bquery;
} else if (bquery_num == 2) {
if (must_bquery == nullptr) {
// should + must_not
binary_query->relation = QueryRelation::R3;
binary_query->left_query->bin = must_not_bquery;
binary_query->right_query->bin = should_bquery;
} else if (should_bquery == nullptr) {
// must + must_not
binary_query->relation = QueryRelation::R4;
binary_query->left_query->bin = must_bquery;
binary_query->right_query->bin = must_not_bquery;
} else {
// must + should
binary_query->relation = QueryRelation::R3;
binary_query->left_query->bin = must_bquery;
binary_query->right_query->bin = should_bquery;
}
} else {
if (must_bquery != nullptr) {
binary_query = must_bquery;
} else if (should_bquery != nullptr) {
binary_query = should_bquery;
} else {
binary_query = must_not_bquery;
}
}
return Status::OK();
}
uint64_t
BinaryQueryHeight(BinaryQueryPtr& binary_query) {
if (binary_query == nullptr) {
return 1;
}
uint64_t left_height = 0, right_height = 0;
if (binary_query->left_query != nullptr) {
left_height = BinaryQueryHeight(binary_query->left_query->bin);
}
if (binary_query->right_query != nullptr) {
right_height = BinaryQueryHeight(binary_query->right_query->bin);
}
return left_height > right_height ? left_height + 1 : right_height + 1;
}
/**
* rules:
* 1. The child node of 'should' and 'must_not' can only be 'term query' and 'range query'.
* 2. One layer cannot include bool query and leaf query.
* 3. The direct child node of 'bool' node cannot be 'should' node or 'must_not' node.
* 4. All filters are pre-filtered(Do structure query first, then use the result to do filtering for vector query).
*
*/
Status
rule_1(BooleanQueryPtr& boolean_query,
std::stack<BooleanQueryPtr>& path_stack) {
auto status = Status::OK();
if (boolean_query != nullptr) {
path_stack.push(boolean_query);
for (const auto& leaf_query : boolean_query->getLeafQueries()) {
if (!leaf_query->vector_placeholder.empty()) {
while (!path_stack.empty()) {
auto query = path_stack.top();
if (query->getOccur() == Occur::SHOULD ||
query->getOccur() == Occur::MUST_NOT) {
std::string msg =
"The child node of 'should' and 'must_not' can "
"only be 'term query' and 'range query'.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
path_stack.pop();
}
}
}
for (auto query : boolean_query->getBooleanQueries()) {
status = rule_1(query, path_stack);
if (!status.ok()) {
return status;
}
}
}
return status;
}
Status
rule_2(BooleanQueryPtr& boolean_query) {
auto status = Status::OK();
if (boolean_query != nullptr) {
if (!boolean_query->getBooleanQueries().empty() &&
!boolean_query->getLeafQueries().empty()) {
std::string msg =
"One layer cannot include bool query and leaf query.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
} else {
for (auto query : boolean_query->getBooleanQueries()) {
status = rule_2(query);
if (!status.ok()) {
return status;
}
}
}
}
return status;
}
Status
ValidateBooleanQuery(BooleanQueryPtr& boolean_query) {
auto status = Status::OK();
if (boolean_query != nullptr) {
for (auto& query : boolean_query->getBooleanQueries()) {
if (query->getOccur() == Occur::SHOULD ||
query->getOccur() == Occur::MUST_NOT) {
std::string msg =
"The direct child node of 'bool' node cannot be 'should' "
"node or 'must_not' node.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
}
std::stack<BooleanQueryPtr> path_stack;
status = rule_1(boolean_query, path_stack);
if (!status.ok()) {
return status;
}
status = rule_2(boolean_query);
if (!status.ok()) {
return status;
}
}
return status;
}
bool
ValidateBinaryQuery(BinaryQueryPtr& binary_query) {
uint64_t height = BinaryQueryHeight(binary_query);
return height > 1;
}
} // namespace query_old
} // namespace milvus

45
internal/core/src/query/deprecated/BinaryQuery.h
View File

@ -1,45 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include "BooleanQuery.h"
namespace milvus {
namespace query_old {
BinaryQueryPtr
ConstructBinTree(std::vector<BooleanQueryPtr> clauses,
QueryRelation relation,
uint64_t idx);
Status
ConstructLeafBinTree(std::vector<LeafQueryPtr> leaf_clauses,
BinaryQueryPtr binary_query,
uint64_t idx);
Status
GenBinaryQuery(BooleanQueryPtr clause, BinaryQueryPtr& binary_query);
uint64_t
BinaryQueryHeight(BinaryQueryPtr& binary_query);
Status
ValidateBooleanQuery(BooleanQueryPtr& boolean_query);
bool
ValidateBinaryQuery(BinaryQueryPtr& binary_query);
} // namespace query_old
} // namespace milvus

87
internal/core/src/query/deprecated/BooleanQuery.h
View File

@ -1,87 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include "GeneralQuery.h"
#include "utils/Status.h"
namespace milvus {
namespace query_old {
enum class Occur {
INVALID = 0,
MUST,
MUST_NOT,
SHOULD,
};
class BooleanQuery {
public:
BooleanQuery() {
}
explicit BooleanQuery(Occur occur) : occur_(occur) {
}
Occur
getOccur() {
return occur_;
}
void
SetOccur(Occur occur) {
occur_ = occur;
}
void
AddBooleanQuery(std::shared_ptr<BooleanQuery> boolean_clause) {
boolean_clauses_.emplace_back(boolean_clause);
}
void
AddLeafQuery(LeafQueryPtr leaf_query) {
leaf_queries_.emplace_back(leaf_query);
}
void
SetLeafQuery(std::vector<LeafQueryPtr> leaf_queries) {
leaf_queries_ = leaf_queries;
}
std::vector<std::shared_ptr<BooleanQuery>>
getBooleanQueries() {
return boolean_clauses_;
}
BinaryQueryPtr&
getBinaryQuery() {
return binary_query_;
}
std::vector<LeafQueryPtr>&
getLeafQueries() {
return leaf_queries_;
}
private:
Occur occur_ = Occur::INVALID;
std::vector<std::shared_ptr<BooleanQuery>> boolean_clauses_;
std::vector<LeafQueryPtr> leaf_queries_;
BinaryQueryPtr binary_query_ = std::make_shared<BinaryQuery>();
};
using BooleanQueryPtr = std::shared_ptr<BooleanQuery>;
} // namespace query_old
} // namespace milvus

148
internal/core/src/query/deprecated/GeneralQuery.h
View File

@ -1,148 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <iostream>
#include <memory>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#include "common/Types.h"
#include "utils/Json.h"
namespace milvus {
namespace query_old {
enum class CompareOperator {
LT = 0,
LTE,
EQ,
GT,
GTE,
NE,
};
enum class QueryRelation {
INVALID = 0,
R1,
R2,
R3,
R4,
AND,
OR,
};
struct QueryColumn {
std::string name;
std::string column_value;
};
struct TermQuery {
milvus::json json_obj;
// std::string field_name;
// std::vector<uint8_t> field_value;
// float boost;
};
using TermQueryPtr = std::shared_ptr<TermQuery>;
struct CompareExpr {
CompareOperator compare_operator;
std::string operand;
};
struct RangeQuery {
milvus::json json_obj;
// std::string field_name;
// std::vector<CompareExpr> compare_expr;
// float boost;
};
using RangeQueryPtr = std::shared_ptr<RangeQuery>;
struct VectorRecord {
size_t vector_count;
std::vector<float> float_data;
std::vector<uint8_t> binary_data;
};
struct VectorQuery {
std::string field_name;
milvus::json extra_params = {};
int64_t topk;
int64_t nq;
std::string metric_type = "";
float boost;
VectorRecord query_vector;
};
using VectorQueryPtr = std::shared_ptr<VectorQuery>;
struct LeafQuery;
using LeafQueryPtr = std::shared_ptr<LeafQuery>;
struct BinaryQuery;
using BinaryQueryPtr = std::shared_ptr<BinaryQuery>;
struct GeneralQuery {
LeafQueryPtr leaf;
BinaryQueryPtr bin = std::make_shared<BinaryQuery>();
};
using GeneralQueryPtr = std::shared_ptr<GeneralQuery>;
struct LeafQuery {
TermQueryPtr term_query;
RangeQueryPtr range_query;
std::string vector_placeholder;
float query_boost;
};
struct BinaryQuery {
GeneralQueryPtr left_query;
GeneralQueryPtr right_query;
QueryRelation relation;
float query_boost;
bool is_not = false;
};
struct Query {
GeneralQueryPtr root;
std::unordered_map<std::string, VectorQueryPtr> vectors;
std::string collection_id;
std::vector<std::string> partitions;
std::vector<std::string> field_names;
std::set<std::string> index_fields;
std::unordered_map<std::string, std::string> metric_types;
std::string index_type;
};
using QueryPtr = std::shared_ptr<Query>;
} // namespace query_old
namespace query {
struct QueryDeprecated {
int64_t num_queries; //
int topK; // topK of queries
std::string
field_name; // must be fakevec, whose data_type must be VEC_FLOAT(DIM)
std::vector<float> query_raw_data; // must be size of num_queries * DIM
};
// std::unique_ptr<Query> CreateNaiveQueryPtr(int64_t num_queries, int topK, std::string& field_name, const float*
// raw_data) {
// return std::
//}
using QueryDeprecatedPtr = std::shared_ptr<QueryDeprecated>;
} // namespace query
} // namespace milvus

295
internal/core/src/query/deprecated/ParserDeprecated.cpp
View File

@ -1,295 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <iostream>
#include <random>
#include "ParserDeprecated.h"
namespace milvus::wtf {
using google::protobuf::RepeatedField;
using google::protobuf::RepeatedPtrField;
#if 1
#if 0
void
CopyRowRecords(const RepeatedPtrField<proto::service::PlaceholderValue>& grpc_records,
const RepeatedField<int64_t>& grpc_id_array,
engine::VectorsData& vectors
) {
// step 1: copy vector data
int64_t float_data_size = 0, binary_data_size = 0;
for (auto& record : grpc_records) {
float_data_size += record.float_data_size();
binary_data_size += record.binary_data().size();
}
std::vector<float> float_array(float_data_size, 0.0f);
std::vector<uint8_t> binary_array(binary_data_size, 0);
int64_t offset = 0;
if (float_data_size > 0) {
for (auto& record : grpc_records) {
memcpy(&float_array[offset], record.float_data().data(), record.float_data_size() * sizeof(float));
offset += record.float_data_size();
}
} else if (binary_data_size > 0) {
for (auto& record : grpc_records) {
memcpy(&binary_array[offset], record.binary_data().data(), record.binary_data().size());
offset += record.binary_data().size();
}
}
// step 2: copy id array
std::vector<int64_t> id_array;
if (grpc_id_array.size() > 0) {
id_array.resize(grpc_id_array.size());
memcpy(id_array.data(), grpc_id_array.data(), grpc_id_array.size() * sizeof(int64_t));
}
// step 3: construct vectors
vectors.vector_count_ = grpc_records.size();
vectors.float_data_.swap(float_array);
vectors.binary_data_.swap(binary_array);
vectors.id_array_.swap(id_array);
}
#endif
Status
ProcessLeafQueryJson(const milvus::json& query_json,
query_old::BooleanQueryPtr& query,
std::string& field_name) {
#if 1
if (query_json.contains("term")) {
auto leaf_query = std::make_shared<query_old::LeafQuery>();
auto term_query = std::make_shared<query_old::TermQuery>();
milvus::json json_obj = query_json["term"];
JSON_NULL_CHECK(json_obj);
JSON_OBJECT_CHECK(json_obj);
term_query->json_obj = json_obj;
milvus::json::iterator json_it = json_obj.begin();
field_name = json_it.key();
leaf_query->term_query = term_query;
query->AddLeafQuery(leaf_query);
} else if (query_json.contains("range")) {
auto leaf_query = std::make_shared<query_old::LeafQuery>();
auto range_query = std::make_shared<query_old::RangeQuery>();
milvus::json json_obj = query_json["range"];
JSON_NULL_CHECK(json_obj);
JSON_OBJECT_CHECK(json_obj);
range_query->json_obj = json_obj;
milvus::json::iterator json_it = json_obj.begin();
field_name = json_it.key();
leaf_query->range_query = range_query;
query->AddLeafQuery(leaf_query);
} else if (query_json.contains("vector")) {
auto leaf_query = std::make_shared<query_old::LeafQuery>();
auto vector_json = query_json["vector"];
JSON_NULL_CHECK(vector_json);
leaf_query->vector_placeholder = vector_json.get<std::string>();
query->AddLeafQuery(leaf_query);
} else {
return Status{SERVER_INVALID_ARGUMENT, "Leaf query get wrong key"};
}
#endif
return Status::OK();
}
Status
ProcessBooleanQueryJson(const milvus::json& query_json,
query_old::BooleanQueryPtr& boolean_query,
query_old::QueryPtr& query_ptr) {
#if 1
if (query_json.empty()) {
return Status{SERVER_INVALID_ARGUMENT, "BoolQuery is null"};
}
for (auto& el : query_json.items()) {
if (el.key() == "must") {
boolean_query->SetOccur(query_old::Occur::MUST);
auto must_json = el.value();
if (!must_json.is_array()) {
std::string msg = "Must json string is not an array";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
for (auto& json : must_json) {
auto must_query = std::make_shared<query_old::BooleanQuery>();
if (json.contains("must") || json.contains("should") ||
json.contains("must_not")) {
STATUS_CHECK(
ProcessBooleanQueryJson(json, must_query, query_ptr));
boolean_query->AddBooleanQuery(must_query);
} else {
std::string field_name;
STATUS_CHECK(
ProcessLeafQueryJson(json, boolean_query, field_name));
if (!field_name.empty()) {
query_ptr->index_fields.insert(field_name);
}
}
}
} else if (el.key() == "should") {
boolean_query->SetOccur(query_old::Occur::SHOULD);
auto should_json = el.value();
if (!should_json.is_array()) {
std::string msg = "Should json string is not an array";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
for (auto& json : should_json) {
auto should_query = std::make_shared<query_old::BooleanQuery>();
if (json.contains("must") || json.contains("should") ||
json.contains("must_not")) {
STATUS_CHECK(
ProcessBooleanQueryJson(json, should_query, query_ptr));
boolean_query->AddBooleanQuery(should_query);
} else {
std::string field_name;
STATUS_CHECK(
ProcessLeafQueryJson(json, boolean_query, field_name));
if (!field_name.empty()) {
query_ptr->index_fields.insert(field_name);
}
}
}
} else if (el.key() == "must_not") {
boolean_query->SetOccur(query_old::Occur::MUST_NOT);
auto should_json = el.value();
if (!should_json.is_array()) {
std::string msg = "Must_not json string is not an array";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
for (auto& json : should_json) {
if (json.contains("must") || json.contains("should") ||
json.contains("must_not")) {
auto must_not_query =
std::make_shared<query_old::BooleanQuery>();
STATUS_CHECK(ProcessBooleanQueryJson(
json, must_not_query, query_ptr));
boolean_query->AddBooleanQuery(must_not_query);
} else {
std::string field_name;
STATUS_CHECK(
ProcessLeafQueryJson(json, boolean_query, field_name));
if (!field_name.empty()) {
query_ptr->index_fields.insert(field_name);
}
}
}
} else {
std::string msg =
"BoolQuery json string does not include bool query";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
}
#endif
return Status::OK();
}
Status
DeserializeJsonToBoolQuery(const google::protobuf::RepeatedPtrField<
::milvus::grpc::VectorParam>& vector_params,
const std::string_view dsl_string,
query_old::BooleanQueryPtr& boolean_query,
query_old::QueryPtr& query_ptr) {
#if 1
try {
milvus::json dsl_json = Json::parse(dsl_string);
if (dsl_json.empty()) {
return Status{SERVER_INVALID_ARGUMENT, "Query dsl is null"};
}
auto status = Status::OK();
if (vector_params.empty()) {
return Status(SERVER_INVALID_DSL_PARAMETER,
"DSL must include vector query");
}
for (const auto& vector_param : vector_params) {
const std::string_view vector_string = vector_param.json();
milvus::json vector_json = Json::parse(vector_string);
milvus::json::iterator it = vector_json.begin();
std::string placeholder = it.key();
auto vector_query = std::make_shared<query_old::VectorQuery>();
milvus::json::iterator vector_param_it = it.value().begin();
if (vector_param_it != it.value().end()) {
const std::string_view field_name = vector_param_it.key();
vector_query->field_name = field_name;
milvus::json param_json = vector_param_it.value();
int64_t topk = param_json["topk"];
// STATUS_CHECK(server::ValidateSearchTopk(topk));
vector_query->topk = topk;
if (param_json.contains("metric_type")) {
std::string metric_type = param_json["metric_type"];
vector_query->metric_type = metric_type;
query_ptr->metric_types.insert(
{field_name, param_json["metric_type"]});
}
if (!vector_param_it.value()["params"].empty()) {
vector_query->extra_params =
vector_param_it.value()["params"];
}
query_ptr->index_fields.insert(field_name);
}
engine::VectorsData vector_data;
CopyRowRecords(
vector_param.row_record().records(),
google::protobuf::RepeatedField<google::protobuf::int64>(),
vector_data);
vector_query->query_vector.vector_count = vector_data.vector_count_;
vector_query->query_vector.binary_data.swap(
vector_data.binary_data_);
vector_query->query_vector.float_data.swap(vector_data.float_data_);
query_ptr->vectors.insert(
std::make_pair(placeholder, vector_query));
}
if (dsl_json.contains("bool")) {
auto boolean_query_json = dsl_json["bool"];
JSON_NULL_CHECK(boolean_query_json);
status = ProcessBooleanQueryJson(
boolean_query_json, boolean_query, query_ptr);
if (!status.ok()) {
return Status(SERVER_INVALID_DSL_PARAMETER,
"DSL does not include bool");
}
} else {
return Status(SERVER_INVALID_DSL_PARAMETER,
"DSL does not include bool query");
}
return Status::OK();
} catch (std::exception& e) {
return Status(SERVER_INVALID_DSL_PARAMETER, e.what());
}
#endif
return Status::OK();
}
#endif
query_old::QueryPtr
Transformer(proto::service::Query* request) {
query_old::BooleanQueryPtr boolean_query =
std::make_shared<query_old::BooleanQuery>();
query_old::QueryPtr query_ptr = std::make_shared<query_old::Query>();
#if 0
query_ptr->collection_id = request->collection_name();
auto status = DeserializeJsonToBoolQuery(request->placeholders(), request->dsl(), boolean_query, query_ptr);
status = query_old::ValidateBooleanQuery(boolean_query);
query_old::GeneralQueryPtr general_query = std::make_shared<query_old::GeneralQuery>();
query_old::GenBinaryQuery(boolean_query, general_query->bin);
query_ptr->root = general_query;
#endif
return query_ptr;
}
} // namespace milvus::wtf

23
internal/core/src/query/deprecated/ParserDeprecated.h
View File

@ -1,23 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include "pb/milvus.pb.h"
#include "query/deprecated/BooleanQuery.h"
#include "query/deprecated/BinaryQuery.h"
#include "query/deprecated/GeneralQuery.h"
namespace milvus::wtf {
query_old::QueryPtr
Transformer(proto::milvus::SearchRequest* query);
} // namespace milvus::wtf

523
internal/core/src/query/deprecated/ValidationUtil.cpp
View File

@ -1,523 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "ValidationUtil.h"
#include "config/ServerConfig.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <algorithm>
#include <limits>
#include <set>
#include <string>
namespace milvus {
namespace server {
namespace {
Status
CheckParameterRange(const milvus::json& json_params,
const std::string_view param_name,
int64_t min,
int64_t max,
bool min_close = true,
bool max_closed = true) {
if (json_params.find(param_name) == json_params.end()) {
std::string msg = "Parameter list must contain: ";
msg += param_name;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
try {
int64_t value = json_params[param_name];
bool min_err = min_close ? value < min : value <= min;
bool max_err = max_closed ? value > max : value >= max;
if (min_err || max_err) {
std::string msg = "Invalid " + param_name +
" value: " + std::to_string(value) +
". Valid range is " + (min_close ? "[" : "(") +
std::to_string(min) + ", " + std::to_string(max) +
(max_closed ? "]" : ")");
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
} catch (std::exception& e) {
std::string msg = "Invalid " + param_name + ": ";
msg += e.what();
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
CheckParameterExistence(const milvus::json& json_params,
const std::string_view param_name) {
if (json_params.find(param_name) == json_params.end()) {
std::string msg = "Parameter list must contain: ";
msg += param_name;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
try {
int64_t value = json_params[param_name];
if (value < 0) {
std::string msg =
"Invalid " + param_name + " value: " + std::to_string(value);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
} catch (std::exception& e) {
std::string msg = "Invalid " + param_name + ": ";
msg += e.what();
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
} // namespace
Status
ValidateCollectionName(const std::string_view collection_name) {
// Collection name shouldn't be empty.
if (collection_name.empty()) {
std::string msg = "Collection name should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_NAME, msg);
}
std::string invalid_msg =
"Invalid collection name: " + collection_name + ". ";
// Collection name size shouldn't exceed engine::MAX_NAME_LENGTH.
if (collection_name.size() > engine::MAX_NAME_LENGTH) {
std::string msg =
invalid_msg + "The length of a collection name must be less than " +
std::to_string(engine::MAX_NAME_LENGTH) + " characters.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_NAME, msg);
}
// Collection name first character should be underscore or character.
char first_char = collection_name[0];
if (first_char != '_' && std::isalpha(first_char) == 0) {
std::string msg = invalid_msg +
"The first character of a collection name must be an "
"underscore or letter.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_NAME, msg);
}
int64_t table_name_size = collection_name.size();
for (int64_t i = 1; i < table_name_size; ++i) {
char name_char = collection_name[i];
if (name_char != '_' && name_char != '$' &&
std::isalnum(name_char) == 0) {
std::string msg = invalid_msg +
"Collection name can only contain numbers, "
"letters, and underscores.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_NAME, msg);
}
}
return Status::OK();
}
Status
ValidateFieldName(const std::string_view field_name) {
// Field name shouldn't be empty.
if (field_name.empty()) {
std::string msg = "Field name should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
std::string invalid_msg = "Invalid field name: " + field_name + ". ";
// Field name size shouldn't exceed engine::MAX_NAME_LENGTH.
if (field_name.size() > engine::MAX_NAME_LENGTH) {
std::string msg =
invalid_msg + "The length of a field name must be less than " +
std::to_string(engine::MAX_NAME_LENGTH) + " characters.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
// Field name first character should be underscore or character.
char first_char = field_name[0];
if (first_char != '_' && std::isalpha(first_char) == 0) {
std::string msg = invalid_msg +
"The first character of a field name must be an "
"underscore or letter.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
int64_t field_name_size = field_name.size();
for (int64_t i = 1; i < field_name_size; ++i) {
char name_char = field_name[i];
if (name_char != '_' && std::isalnum(name_char) == 0) {
std::string msg = invalid_msg +
"Field name cannot only contain numbers, "
"letters, and underscores.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
}
return Status::OK();
}
Status
ValidateVectorIndexType(std::string& index_type, bool is_binary) {
// Index name shouldn't be empty.
if (index_type.empty()) {
std::string msg = "Index type should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
// string case insensitive
std::transform(
index_type.begin(), index_type.end(), index_type.begin(), ::toupper);
static std::set<std::string> s_vector_index_type = {
knowhere::IndexEnum::INVALID,
knowhere::IndexEnum::INDEX_FAISS_IDMAP,
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
knowhere::IndexEnum::INDEX_FAISS_IVFPQ,
knowhere::IndexEnum::INDEX_FAISS_IVFSQ8,
knowhere::IndexEnum::INDEX_HNSW,
};
static std::set<std::string> s_binary_index_types = {
knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
};
std::set<std::string>& index_types =
is_binary ? s_binary_index_types : s_vector_index_type;
if (index_types.find(index_type) == index_types.end()) {
std::string msg = "Invalid index type: " + index_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_INDEX_TYPE, msg);
}
return Status::OK();
}
Status
ValidateStructuredIndexType(std::string& index_type) {
// Index name shouldn't be empty.
if (index_type.empty()) {
std::string msg = "Index type should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_FIELD_NAME, msg);
}
// string case insensitive
std::transform(
index_type.begin(), index_type.end(), index_type.begin(), ::toupper);
static std::set<std::string> s_index_types = {
engine::DEFAULT_STRUCTURED_INDEX,
};
if (s_index_types.find(index_type) == s_index_types.end()) {
std::string msg = "Invalid index type: " + index_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_INDEX_TYPE, msg);
}
return Status::OK();
}
Status
ValidateDimension(int64_t dim, bool is_binary) {
if (dim <= 0 || dim > engine::MAX_DIMENSION) {
std::string msg = "Invalid dimension: " + std::to_string(dim) +
". Should be in range 1 ~ " +
std::to_string(engine::MAX_DIMENSION) + ".";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
}
if (is_binary && (dim % 8) != 0) {
std::string msg = "Invalid dimension: " + std::to_string(dim) +
". Should be multiple of 8.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
}
return Status::OK();
}
Status
ValidateIndexParams(const milvus::json& index_params,
int64_t dimension,
const std::string_view index_type) {
if (engine::utils::IsFlatIndexType(index_type)) {
return Status::OK();
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT ||
index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
auto status = CheckParameterRange(
index_params, knowhere::IndexParams::nlist, 1, 65536);
if (!status.ok()) {
return status;
}
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
auto status = CheckParameterRange(
index_params, knowhere::IndexParams::nlist, 1, 65536);
if (!status.ok()) {
return status;
}
status =
CheckParameterExistence(index_params, knowhere::IndexParams::m);
if (!status.ok()) {
return status;
}
// special check for 'm' parameter
int64_t m_value = index_params[knowhere::IndexParams::m];
if (!milvus::knowhere::IVFPQConfAdapter::GetValidCPUM(dimension,
m_value)) {
std::string msg = "Invalid m, dimension can't not be divided by m ";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
/*std::vector<int64_t> resset;
milvus::knowhere::IVFPQConfAdapter::GetValidMList(dimension, resset);
int64_t m_value = index_params[knowhere::IndexParams::m];
if (resset.empty()) {
std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg);
}
auto iter = std::find(std::begin(resset), std::end(resset), m_value);
if (iter == std::end(resset)) {
std::string msg =
"Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: ";
for (size_t i = 0; i < resset.size(); i++) {
if (i != 0) {
msg += ",";
}
msg += std::to_string(resset[i]);
}
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}*/
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
auto status =
CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(
index_params, knowhere::IndexParams::efConstruction, 8, 512);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
Status
ValidateSegmentRowCount(int64_t segment_row_count) {
int64_t min = config.engine.build_index_threshold();
int max = engine::MAX_SEGMENT_ROW_COUNT;
if (segment_row_count < min || segment_row_count > max) {
std::string msg =
"Invalid segment row count: " + std::to_string(segment_row_count) +
". " + "Should be in range " + std::to_string(min) + " ~ " +
std::to_string(max) + ".";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_SEGMENT_ROW_COUNT, msg);
}
return Status::OK();
}
Status
ValidateIndexMetricType(const std::string_view metric_type,
const std::string_view index_type) {
if (engine::utils::IsFlatIndexType(index_type)) {
// pass
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
// binary
if (metric_type != knowhere::Metric::HAMMING &&
metric_type != knowhere::Metric::JACCARD) {
std::string msg = "Index metric type " + metric_type +
" does not match index type " + index_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
} else {
// float
if (metric_type != knowhere::Metric::L2 &&
metric_type != knowhere::Metric::IP) {
std::string msg = "Index metric type " + metric_type +
" does not match index type " + index_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
return Status::OK();
}
Status
ValidateSearchMetricType(const std::string_view metric_type, bool is_binary) {
if (is_binary) {
// binary
if (metric_type == knowhere::Metric::L2 ||
metric_type == knowhere::Metric::IP) {
std::string msg =
"Cannot search binary entities with index metric type " +
metric_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
} else {
// float
if (metric_type == knowhere::Metric::HAMMING ||
metric_type == knowhere::Metric::JACCARD) {
std::string msg =
"Cannot search float entities with index metric type " +
metric_type;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
return Status::OK();
}
Status
ValidateSearchTopk(int64_t top_k) {
if (top_k <= 0 || top_k > QUERY_MAX_TOPK) {
std::string msg = "Invalid topk: " + std::to_string(top_k) + ". " +
"The topk must be within the range of 1 ~ 16384.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_TOPK, msg);
}
return Status::OK();
}
Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
for (const std::string_view tag : partition_tags) {
// Partition nametag shouldn't be empty.
if (tag.empty()) {
std::string msg = "Partition tag should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
std::string invalid_msg = "Invalid partition tag: " + tag + ". ";
// Partition tag size shouldn't exceed 255.
if (tag.size() > engine::MAX_NAME_LENGTH) {
std::string msg = invalid_msg +
"The length of a partition tag must be less than "
"255 characters.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
// Partition tag first character should be underscore or character.
char first_char = tag[0];
if (first_char != '_' && std::isalnum(first_char) == 0) {
std::string msg = invalid_msg +
"The first character of a partition tag must be "
"an underscore or letter.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
int64_t tag_size = tag.size();
for (int64_t i = 1; i < tag_size; ++i) {
char name_char = tag[i];
if (name_char != '_' && name_char != '$' &&
std::isalnum(name_char) == 0) {
std::string msg = invalid_msg +
"Partition tag can only contain numbers, "
"letters, and underscores.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
}
#if 0
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
std::string valid_tag = tag;
StringHelpFunctions::TrimStringBlank(valid_tag);
if (valid_tag.empty()) {
std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
// max length of partition tag
if (valid_tag.length() > engine::MAX_NAME_LENGTH) {
std::string msg = "Invalid partition tag: " + valid_tag + ". " +
"Partition tag exceed max length: " + std::to_string(engine::MAX_NAME_LENGTH);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_PARTITION_TAG, msg);
}
#endif
}
return Status::OK();
}
Status
ValidateInsertDataSize(const InsertParam& insert_param) {
int64_t chunk_size = 0;
for (auto& pair : insert_param.fields_data_) {
for (auto& data : pair.second) {
chunk_size += data.second;
}
}
if (chunk_size > engine::MAX_INSERT_DATA_SIZE) {
std::string msg =
"The amount of data inserted each time cannot exceed " +
std::to_string(engine::MAX_INSERT_DATA_SIZE / engine::MB) + " MB";
return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg);
}
return Status::OK();
}
Status
ValidateCompactThreshold(double threshold) {
if (threshold > 1.0 || threshold < 0.0) {
std::string msg =
"Invalid compact threshold: " + std::to_string(threshold) +
". Should be in range [0.0, 1.0]";
return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg);
}
return Status::OK();
}
} // namespace server
} // namespace milvus

71
internal/core/src/query/deprecated/ValidationUtil.h
View File

@ -1,71 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "db/Types.h"
#include "server/delivery/request/Types.h"
#include "utils/Json.h"
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
namespace server {
constexpr int64_t QUERY_MAX_TOPK = 16384;
constexpr int64_t GPU_QUERY_MAX_TOPK = 2048;
constexpr int64_t GPU_QUERY_MAX_NPROBE = 2048;
extern Status
ValidateCollectionName(const std::string_view collection_name);
extern Status
ValidateFieldName(const std::string_view field_name);
extern Status
ValidateDimension(int64_t dimension, bool is_binary);
extern Status
ValidateVectorIndexType(std::string& index_type, bool is_binary);
extern Status
ValidateStructuredIndexType(std::string& index_type);
extern Status
ValidateIndexParams(const milvus::json& index_params,
int64_t dimension,
const std::string_view index_type);
extern Status
ValidateSegmentRowCount(int64_t segment_row_count);
extern Status
ValidateIndexMetricType(const std::string_view metric_type,
const std::string_view index_type);
extern Status
ValidateSearchMetricType(const std::string_view metric_type, bool is_binary);
extern Status
ValidateSearchTopk(int64_t top_k);
extern Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags);
extern Status
ValidateInsertDataSize(const InsertParam& insert_param);
extern Status
ValidateCompactThreshold(double threshold);
} // namespace server
} // namespace milvus

2
internal/core/src/query/generated/ExecPlanNodeVisitor.h
View File

@ -12,7 +12,7 @@
#pragma once
// Generated File
// DO NOT EDIT
#include "utils/Json.h"
#include "common/Json.h"
#include "query/PlanImpl.h"
#include "segcore/SegmentGrowing.h"
#include <utility>

2
internal/core/src/query/generated/ShowExprVisitor.h
View File

@ -77,6 +77,6 @@ class ShowExprVisitor : public ExprVisitor {
}
private:
std::optional<json> json_opt_;
std::optional<nlohmann::json> json_opt_;
};
} // namespace milvus::query

4
internal/core/src/query/generated/ShowPlanNodeVisitor.h
View File

@ -12,8 +12,8 @@
#pragma once
// Generated File
// DO NOT EDIT
#include "exceptions/EasyAssert.h"
#include "utils/Json.h"
#include "common/EasyAssert.h"
#include "common/Json.h"
#include <optional>
#include <utility>

2
internal/core/src/query/generated/VerifyPlanNodeVisitor.h
View File

@ -12,7 +12,7 @@
#pragma once
// Generated File
// DO NOT EDIT
#include "utils/Json.h"
#include "common/Json.h"
#include "query/PlanImpl.h"
#include "segcore/SegmentGrowing.h"
#include <utility>

2
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -27,7 +27,7 @@
#include "arrow/type_fwd.h"
#include "common/Json.h"
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "pb/plan.pb.h"
#include "query/ExprImpl.h"
#include "query/Relational.h"

2
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -17,7 +17,7 @@
#include "query/SubSearchResult.h"
#include "query/generated/ExecExprVisitor.h"
#include "segcore/SegmentGrowing.h"
#include "utils/Json.h"
#include "common/Json.h"
#include "log/Log.h"
namespace milvus::query {

4
internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
View File

@ -11,10 +11,10 @@
#include <optional>
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/Json.h"
#include "query/generated/ShowExprVisitor.h"
#include "query/generated/ShowPlanNodeVisitor.h"
#include "utils/Json.h"
namespace milvus::query {
#if 0

2
internal/core/src/segcore/ConcurrentVector.h
View File

@ -30,7 +30,7 @@
#include "common/Span.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "storage/FieldData.h"
namespace milvus::segcore {

2
internal/core/src/segcore/Reduce.cpp
View File

@ -19,7 +19,7 @@
#include "SegmentInterface.h"
#include "Utils.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "pkVisitor.h"
namespace milvus::segcore {

9
internal/core/src/segcore/Reduce.h
View File

@ -18,7 +18,6 @@
#include <queue>
#include <unordered_set>
#include "utils/Status.h"
#include "common/type_c.h"
#include "common/QueryResult.h"
#include "query/PlanImpl.h"
@ -84,15 +83,15 @@ class ReduceHelper {
GetSearchResultDataSlice(int slice_index_);
private:
std::vector<int64_t> slice_topKs_;
std::vector<SearchResult*>& search_results_;
milvus::query::Plan* plan_;
std::vector<int64_t> slice_nqs_;
std::vector<int64_t> slice_topKs_;
int64_t total_nq_;
int64_t num_segments_;
int64_t num_slices_;
milvus::query::Plan* plan_;
std::vector<SearchResult*>& search_results_;
std::vector<int64_t> slice_nqs_prefix_sum_;
// dim0: num_segments_; dim1: total_nq_; dim2: offset

2
internal/core/src/segcore/ScalarIndex.cpp
View File

@ -9,7 +9,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "ScalarIndex.h"
namespace milvus::segcore {

2
internal/core/src/segcore/SealedIndexingRecord.h
View File

@ -19,7 +19,7 @@
#include <tbb/concurrent_hash_map.h>
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "index/VectorIndex.h"
namespace milvus::segcore {

2
internal/core/src/segcore/SegcoreConfig.cpp
View File

@ -11,7 +11,7 @@
#include "common/Schema.h"
#include "SegcoreConfig.h"
#include "utils/Json.h"
#include "common/Json.h"
#include "yaml-cpp/yaml.h"
namespace milvus::segcore {

4
internal/core/src/segcore/SegcoreConfig.h
View File

@ -15,9 +15,9 @@
#include <string>
#include "common/Types.h"
#include "common/Json.h"
#include "index/Utils.h"
#include "exceptions/EasyAssert.h"
#include "utils/Json.h"
#include "common/EasyAssert.h"
namespace milvus::segcore {

1
internal/core/src/segcore/SegmentGrowing.h
View File

@ -18,7 +18,6 @@
#include "common/Schema.h"
#include "common/Types.h"
#include "query/Plan.h"
#include "query/deprecated/GeneralQuery.h"
#include "segcore/SegmentInterface.h"
namespace milvus::segcore {

5
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -19,6 +19,7 @@
#include <type_traits>
#include "common/Consts.h"
#include "common/EasyAssert.h"
#include "common/Types.h"
#include "nlohmann/json.hpp"
#include "query/PlanNode.h"
@ -222,7 +223,7 @@ SegmentGrowingImpl::LoadFieldData(const LoadFieldDataInfo& infos) {
reserved_offset + num_rows);
}
Status
SegcoreError
SegmentGrowingImpl::Delete(int64_t reserved_begin,
int64_t size,
const IdArray* ids,
@ -250,7 +251,7 @@ SegmentGrowingImpl::Delete(int64_t reserved_begin,
// step 2: fill delete record
deleted_record_.push(sort_pks, sort_timestamps.data());
return Status::OK();
return SegcoreError::success();
}
int64_t

6
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -29,10 +29,8 @@
#include "SealedIndexingRecord.h"
#include "SegmentGrowing.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "query/PlanNode.h"
#include "query/deprecated/GeneralQuery.h"
#include "utils/Status.h"
#include "common/IndexMeta.h"
namespace milvus::segcore {
@ -50,7 +48,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
const InsertData* insert_data) override;
// TODO: add id into delete log, possibly bitmap
Status
SegcoreError
Delete(int64_t reserved_offset,
int64_t size,
const IdArray* pks,

2
internal/core/src/segcore/SegmentInterface.h
View File

@ -83,7 +83,7 @@ class SegmentInterface {
// virtual int64_t
// PreDelete(int64_t size) = 0;
virtual Status
virtual SegcoreError
Delete(int64_t reserved_offset,
int64_t size,
const IdArray* pks,

8
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -24,7 +24,7 @@
#include "Utils.h"
#include "Types.h"
#include "common/Json.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "mmap/Column.h"
#include "common/Consts.h"
#include "common/FieldMeta.h"
@ -36,7 +36,7 @@
#include "storage/FieldData.h"
#include "storage/Util.h"
#include "storage/ThreadPools.h"
#include "utils/File.h"
#include "common/File.h"
#include "common/Tracer.h"
namespace milvus::segcore {
@ -980,7 +980,7 @@ SegmentSealedImpl::search_ids(const IdArray& id_array,
return {std::move(res_id_arr), std::move(res_offsets)};
}
Status
SegcoreError
SegmentSealedImpl::Delete(int64_t reserved_offset, // deprecated
int64_t size,
const IdArray* ids,
@ -1007,7 +1007,7 @@ SegmentSealedImpl::Delete(int64_t reserved_offset, // deprecated
}
deleted_record_.push(sort_pks, sort_timestamps.data());
return Status::OK();
return SegcoreError::success();
}
std::string

3
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -28,6 +28,7 @@
#include "SealedIndexingRecord.h"
#include "SegmentSealed.h"
#include "TimestampIndex.h"
#include "common/EasyAssert.h"
#include "mmap/Column.h"
#include "index/ScalarIndex.h"
#include "sys/mman.h"
@ -104,7 +105,7 @@ class SegmentSealedImpl : public SegmentSealed {
std::string
debug() const override;
Status
SegcoreError
Delete(int64_t reserved_offset,
int64_t size,
const IdArray* pks,

7
internal/core/src/segcore/load_field_data_c.cpp
View File

@ -14,7 +14,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/CGoHelper.h"
#include "common/LoadInfo.h"
#include "segcore/load_field_data_c.h"
@ -25,7 +24,7 @@ NewLoadFieldDataInfo(CLoadFieldDataInfo* c_load_field_data_info) {
*c_load_field_data_info = load_field_data_info.release();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -51,7 +50,7 @@ AppendLoadFieldInfo(CLoadFieldDataInfo c_load_field_data_info,
load_field_data_info->field_infos[field_id] = binlog_info;
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -71,7 +70,7 @@ AppendLoadFieldDataPath(CLoadFieldDataInfo c_load_field_data_info,
file_path);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}

38
internal/core/src/segcore/load_index_c.cpp
View File

@ -12,7 +12,7 @@
#include "segcore/load_index_c.h"
#include "common/FieldMeta.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "index/Index.h"
#include "index/IndexFactory.h"
#include "index/Meta.h"
@ -31,12 +31,12 @@ NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info) {
*c_load_index_info = load_index_info.release();
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -60,12 +60,12 @@ AppendIndexParam(CLoadIndexInfo c_load_index_info,
load_index_info->index_params[index_key] = index_value;
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -90,12 +90,12 @@ AppendFieldInfo(CLoadIndexInfo c_load_index_info,
load_index_info->mmap_dir_path = std::string(mmap_dir_path);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -151,12 +151,12 @@ appendVecIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
index_info, file_manager);
load_index_info->index->Load(*binary_set, config);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -184,12 +184,12 @@ appendScalarIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
nullptr);
load_index_info->index->Load(*binary_set);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -269,12 +269,12 @@ AppendIndexV2(CLoadIndexInfo c_load_index_info) {
load_index_info->index->Load(config);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -289,12 +289,12 @@ AppendIndexFilePath(CLoadIndexInfo c_load_index_info, const char* c_file_path) {
load_index_info->index_files.emplace_back(index_file_path);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -313,12 +313,12 @@ AppendIndexInfo(CLoadIndexInfo c_load_index_info,
load_index_info->index_version = version;
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@ -338,12 +338,12 @@ CleanLoadedIndex(CLoadIndexInfo c_load_index_info) {
load_index_info->index_version);
local_chunk_manager->RemoveDir(index_file_path_prefix);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}

15
internal/core/src/segcore/plan_c.cpp
View File

@ -9,7 +9,6 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "common/CGoHelper.h"
#include "pb/segcore.pb.h"
#include "query/Plan.h"
#include "segcore/Collection.h"
@ -28,7 +27,7 @@ CreateSearchPlanByExpr(CCollection c_col,
*col->get_schema(), serialized_expr_plan, size);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
auto plan = (CSearchPlan)res.release();
*res_plan = plan;
@ -41,7 +40,7 @@ CreateSearchPlanByExpr(CCollection c_col,
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
*res_plan = nullptr;
return status;
@ -60,14 +59,14 @@ ParsePlaceholderGroup(CSearchPlan c_plan,
plan, (const uint8_t*)(placeholder_group_blob), blob_size);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
auto group = (CPlaceholderGroup)res.release();
*res_placeholder_group = group;
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
*res_placeholder_group = nullptr;
return status;
@ -94,7 +93,7 @@ GetFieldID(CSearchPlan plan, int64_t* field_id) {
*field_id = milvus::query::GetFieldID(p);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, strdup(e.what()));
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -139,7 +138,7 @@ CreateRetrievePlanByExpr(CCollection c_col,
*col->get_schema(), serialized_expr_plan, size);
auto status = CStatus();
status.error_code = Success;
status.error_code = milvus::Success;
status.error_msg = "";
auto plan = (CRetrievePlan)res.release();
*res_plan = plan;
@ -152,7 +151,7 @@ CreateRetrievePlanByExpr(CCollection c_col,
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
*res_plan = nullptr;
return status;

7
internal/core/src/segcore/reduce_c.cpp
View File

@ -11,9 +11,8 @@
#include <vector>
#include "Reduce.h"
#include "common/CGoHelper.h"
#include "common/QueryResult.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "query/Plan.h"
#include "segcore/reduce_c.h"
#include "segcore/Utils.h"
@ -46,7 +45,7 @@ ReduceSearchResultsAndFillData(CSearchResultDataBlobs* cSearchResultDataBlobs,
*cSearchResultDataBlobs = reduce_helper.GetSearchResultDataBlobs();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -68,7 +67,7 @@ GetSearchResultDataBlob(CProto* searchResultDataBlob,
} catch (std::exception& e) {
searchResultDataBlob->proto_blob = nullptr;
searchResultDataBlob->proto_size = 0;
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}

25
internal/core/src/segcore/segment_c.cpp
View File

@ -12,7 +12,6 @@
#include "segcore/segment_c.h"
#include <memory>
#include "common/CGoHelper.h"
#include "common/LoadInfo.h"
#include "common/Types.h"
#include "common/Tracer.h"
@ -92,7 +91,7 @@ Search(CSegmentInterface c_segment,
milvus::tracer::CloseRootSpan();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -129,7 +128,7 @@ Retrieve(CSegmentInterface c_segment,
span->End();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -191,7 +190,7 @@ Insert(CSegmentInterface c_segment,
reserved_offset, size, row_ids, timestamps, insert_data.get());
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -202,7 +201,7 @@ PreInsert(CSegmentInterface c_segment, int64_t size, int64_t* offset) {
*offset = segment->PreInsert(size);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -222,7 +221,7 @@ Delete(CSegmentInterface c_segment,
segment->Delete(reserved_offset, size, pks.get(), timestamps);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -238,7 +237,7 @@ LoadFieldData(CSegmentInterface c_segment,
segment->LoadFieldData(*load_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -278,7 +277,7 @@ LoadFieldRawData(CSegmentInterface c_segment,
segment->LoadFieldData(milvus::FieldId(field_id), field_data_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -299,7 +298,7 @@ LoadDeletedRecord(CSegmentInterface c_segment,
segment_interface->LoadDeletedRecord(load_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -317,7 +316,7 @@ UpdateSealedSegmentIndex(CSegmentInterface c_segment,
segment->LoadIndex(*load_index_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -334,7 +333,7 @@ UpdateFieldRawDataSize(CSegmentInterface c_segment,
milvus::FieldId(field_id), num_rows, field_data_size);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -349,7 +348,7 @@ DropFieldData(CSegmentInterface c_segment, int64_t field_id) {
segment->DropFieldData(milvus::FieldId(field_id));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}
@ -364,6 +363,6 @@ DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id) {
segment->DropIndex(milvus::FieldId(field_id));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::UnexpectedError, e.what());
}
}

14
internal/core/src/storage/BinlogReader.cpp
View File

@ -15,32 +15,34 @@
// limitations under the License.
#include "storage/BinlogReader.h"
#include "common/EasyAssert.h"
namespace milvus::storage {
Status
milvus::SegcoreError
BinlogReader::Read(int64_t nbytes, void* out) {
auto remain = size_ - tell_;
if (nbytes > remain) {
return Status(SERVER_UNEXPECTED_ERROR, "out range of binlog data");
return SegcoreError(milvus::UnexpectedError,
"out range of binlog data");
}
std::memcpy(out, data_.get() + tell_, nbytes);
tell_ += nbytes;
return Status(SERVER_SUCCESS, "");
return SegcoreError(milvus::Success, "");
}
std::pair<Status, std::shared_ptr<uint8_t[]>>
std::pair<milvus::SegcoreError, std::shared_ptr<uint8_t[]>>
BinlogReader::Read(int64_t nbytes) {
auto remain = size_ - tell_;
if (nbytes > remain) {
return std::make_pair(
Status(SERVER_UNEXPECTED_ERROR, "out range of binlog data"),
SegcoreError(milvus::UnexpectedError, "out range of binlog data"),
nullptr);
}
auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction
auto res = std::shared_ptr<uint8_t[]>(data_.get() + tell_, deleter);
tell_ += nbytes;
return std::make_pair(Status(SERVER_SUCCESS, ""), res);
return std::make_pair(SegcoreError(milvus::Success, ""), res);
}
} // namespace milvus::storage

9
internal/core/src/storage/BinlogReader.h
View File

@ -19,8 +19,7 @@
#include <memory>
#include <utility>
#include "utils/Status.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
namespace milvus::storage {
@ -31,10 +30,10 @@ class BinlogReader {
: data_(binlog_data), size_(length), tell_(0) {
}
Status
SegcoreError
Read(int64_t nbytes, void* out);
std::pair<Status, std::shared_ptr<uint8_t[]>>
std::pair<SegcoreError, std::shared_ptr<uint8_t[]>>
Read(int64_t nbytes);
int64_t
@ -43,9 +42,9 @@ class BinlogReader {
}
private:
std::shared_ptr<uint8_t[]> data_;
int64_t size_;
int64_t tell_;
std::shared_ptr<uint8_t[]> data_;
};
using BinlogReaderPtr = std::shared_ptr<BinlogReader>;

11
internal/core/src/storage/CMakeLists.txt
View File

@ -31,7 +31,7 @@ set(STORAGE_FILES
PayloadReader.cpp
PayloadWriter.cpp
BinlogReader.cpp
IndexData.cpp
IndexData.cpp
InsertData.cpp
Event.cpp
ThreadPool.cpp
@ -45,12 +45,9 @@ set(STORAGE_FILES
add_library(milvus_storage SHARED ${STORAGE_FILES})
find_package(Boost REQUIRED COMPONENTS filesystem)
target_link_libraries(milvus_storage PUBLIC
milvus_common
Boost::filesystem
pthread
target_link_libraries(milvus_storage PUBLIC
milvus_common
pthread
${CONAN_LIBS}
)

2
internal/core/src/storage/DataCodec.cpp
View File

@ -20,7 +20,7 @@
#include "storage/InsertData.h"
#include "storage/IndexData.h"
#include "storage/BinlogReader.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/Consts.h"
namespace milvus::storage {

15
internal/core/src/storage/Event.cpp
View File

@ -15,10 +15,11 @@
// limitations under the License.
#include "storage/Event.h"
#include "nlohmann/json.hpp"
#include "storage/PayloadReader.h"
#include "storage/PayloadWriter.h"
#include "exceptions/EasyAssert.h"
#include "utils/Json.h"
#include "common/EasyAssert.h"
#include "common/Json.h"
#include "common/Consts.h"
#include "common/FieldMeta.h"
@ -138,8 +139,8 @@ DescriptorEventDataFixPart::Serialize() {
DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) {
fix_part = DescriptorEventDataFixPart(reader);
for (auto i = int8_t(EventType::DescriptorEvent);
i < int8_t(EventType::EventTypeEnd);
for (auto i = static_cast<int8_t>(EventType::DescriptorEvent);
i < static_cast<int8_t>(EventType::EventTypeEnd);
i++) {
post_header_lengths.push_back(GetEventFixPartSize(EventType(i)));
}
@ -152,8 +153,8 @@ DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) {
ast = reader->Read(extra_length, extra_bytes.data());
assert(ast.ok());
milvus::json json =
milvus::json::parse(extra_bytes.begin(), extra_bytes.end());
nlohmann::json json =
nlohmann::json::parse(extra_bytes.begin(), extra_bytes.end());
if (json.contains(ORIGIN_SIZE_KEY)) {
extras[ORIGIN_SIZE_KEY] = json[ORIGIN_SIZE_KEY];
}
@ -165,7 +166,7 @@ DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) {
std::vector<uint8_t>
DescriptorEventData::Serialize() {
auto fix_part_data = fix_part.Serialize();
milvus::json extras_json;
nlohmann::json extras_json;
for (auto v : extras) {
extras_json.emplace(v.first, v.second);
}

2
internal/core/src/storage/FieldDataInterface.h
View File

@ -30,7 +30,7 @@
#include "common/FieldMeta.h"
#include "common/Utils.h"
#include "common/VectorTrait.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "storage/Exception.h"
namespace milvus::storage {

2
internal/core/src/storage/IndexData.cpp
View File

@ -15,7 +15,7 @@
// limitations under the License.
#include "storage/IndexData.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/Consts.h"
#include "storage/Event.h"

2
internal/core/src/storage/InsertData.cpp
View File

@ -17,7 +17,7 @@
#include "storage/InsertData.h"
#include "storage/Event.h"
#include "storage/Util.h"
#include "utils/Json.h"
#include "common/Json.h"
#include "common/FieldMeta.h"
#include "common/Consts.h"

2
internal/core/src/storage/MinioChunkManager.cpp
View File

@ -31,7 +31,7 @@
#include "storage/MinioChunkManager.h"
#include "storage/AliyunSTSClient.h"
#include "storage/AliyunCredentialsProvider.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "log/Log.h"
#include "signal.h"

2
internal/core/src/storage/PayloadReader.cpp
View File

@ -15,7 +15,7 @@
// limitations under the License.
#include "storage/PayloadReader.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "storage/Util.h"
#include "parquet/column_reader.h"
#include "arrow/io/api.h"

2
internal/core/src/storage/PayloadStream.cpp
View File

@ -17,7 +17,7 @@
#include "arrow/api.h"
#include "storage/PayloadStream.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
namespace milvus::storage {

2
internal/core/src/storage/PayloadWriter.cpp
View File

@ -15,7 +15,7 @@
// limitations under the License.
#include "storage/PayloadWriter.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/FieldMeta.h"
#include "storage/Util.h"

2
internal/core/src/storage/Util.cpp
View File

@ -18,7 +18,7 @@
#include <memory>
#include "arrow/array/builder_binary.h"
#include "arrow/type_fwd.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
#include "common/Consts.h"
#include "storage/FieldData.h"
#include "storage/FieldDataInterface.h"

63
internal/core/src/storage/parquet_c.cpp
View File

@ -20,7 +20,6 @@
#include "storage/PayloadReader.h"
#include "storage/PayloadWriter.h"
#include "storage/FieldData.h"
#include "common/CGoHelper.h"
#include "storage/Util.h"
using Payload = milvus::storage::Payload;
@ -50,7 +49,8 @@ AddValuesToPayload(CPayloadWriter payloadWriter, const Payload& info) {
p->add_payload(info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -117,7 +117,8 @@ AddOneStringToPayload(CPayloadWriter payloadWriter, char* cstr, int str_size) {
p->add_one_string_payload(cstr, str_size);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -128,7 +129,8 @@ AddOneArrayToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) {
p->add_one_binary_payload(data, length);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -139,7 +141,8 @@ AddOneJSONToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) {
p->add_one_binary_payload(data, length);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -155,7 +158,8 @@ AddBinaryVectorToPayload(CPayloadWriter payloadWriter,
p->add_payload(raw_data_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -173,7 +177,8 @@ AddFloatVectorToPayload(CPayloadWriter payloadWriter,
p->add_payload(raw_data_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -184,7 +189,8 @@ FinishPayloadWriter(CPayloadWriter payloadWriter) {
p->finish();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -240,8 +246,9 @@ NewPayloadReader(int columnType,
break;
}
default: {
return milvus::FailureCStatus(UnexpectedError,
"unsupported data type");
return milvus::FailureCStatus(
milvus::ErrorCodeEnum::UnexpectedError,
"unsupported data type");
}
}
@ -250,7 +257,8 @@ NewPayloadReader(int columnType,
*c_reader = (CPayloadReader)(p.release());
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -262,7 +270,8 @@ GetBoolFromPayload(CPayloadReader payloadReader, int idx, bool* value) {
*value = *reinterpret_cast<const bool*>(field_data->RawValue(idx));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -276,7 +285,8 @@ GetInt8FromPayload(CPayloadReader payloadReader, int8_t** values, int* length) {
reinterpret_cast<int8_t*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -292,7 +302,8 @@ GetInt16FromPayload(CPayloadReader payloadReader,
reinterpret_cast<int16_t*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -308,7 +319,8 @@ GetInt32FromPayload(CPayloadReader payloadReader,
reinterpret_cast<int32_t*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -324,7 +336,8 @@ GetInt64FromPayload(CPayloadReader payloadReader,
reinterpret_cast<int64_t*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -338,7 +351,8 @@ GetFloatFromPayload(CPayloadReader payloadReader, float** values, int* length) {
reinterpret_cast<float*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -354,7 +368,8 @@ GetDoubleFromPayload(CPayloadReader payloadReader,
reinterpret_cast<double*>(const_cast<void*>(field_data->Data()));
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -371,7 +386,8 @@ GetOneStringFromPayload(CPayloadReader payloadReader,
*str_size = field_data->Size(idx);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -388,7 +404,8 @@ GetBinaryVectorFromPayload(CPayloadReader payloadReader,
*length = field_data->get_num_rows();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -405,7 +422,8 @@ GetFloatVectorFromPayload(CPayloadReader payloadReader,
*length = field_data->get_num_rows();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -427,6 +445,7 @@ ReleasePayloadReader(CPayloadReader payloadReader) {
milvus::storage::ReleaseArrowUnused();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}

10
internal/core/src/storage/storage_c.cpp
View File

@ -15,7 +15,6 @@
// limitations under the License.
#include "storage/storage_c.h"
#include "common/CGoHelper.h"
#include "storage/RemoteChunkManagerSingleton.h"
#include "storage/LocalChunkManagerSingleton.h"
@ -33,7 +32,8 @@ GetLocalUsedSize(const char* c_dir, int64_t* size) {
}
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -45,7 +45,8 @@ InitLocalChunkManagerSingleton(const char* c_path) {
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}
@ -74,7 +75,8 @@ InitRemoteChunkManagerSingleton(CStorageConfig c_storage_config) {
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError,
e.what());
}
}

17
internal/core/src/utils/CMakeLists.txt
View File

@ -1,17 +0,0 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
aux_source_directory( ${MILVUS_ENGINE_SRC}/utils UTILS_FILES )
add_library( milvus_utils STATIC ${UTILS_FILES} )

146
internal/core/src/utils/Error.h
View File

@ -1,146 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
#include <exception>
#include <string>
namespace milvus {
using ErrorCode = int32_t;
constexpr ErrorCode SERVER_SUCCESS = 0;
constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000;
constexpr ErrorCode
ToServerErrorCode(const ErrorCode error_code) {
return SERVER_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode DB_SUCCESS = 0;
constexpr ErrorCode DB_ERROR_CODE_BASE = 40000;
constexpr ErrorCode
ToDbErrorCode(const ErrorCode error_code) {
return DB_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode KNOWHERE_SUCCESS = 0;
constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000;
constexpr ErrorCode
ToKnowhereErrorCode(const ErrorCode error_code) {
return KNOWHERE_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode WAL_SUCCESS = 0;
constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000;
constexpr ErrorCode
ToWalErrorCode(const ErrorCode error_code) {
return WAL_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode SS_SUCCESS = 0;
constexpr ErrorCode SS_ERROR_CODE_BASE = 70000;
constexpr ErrorCode
ToSSErrorCode(const ErrorCode error_code) {
return SS_ERROR_CODE_BASE + error_code;
}
// server error code
constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1);
constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2);
constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4);
constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5);
constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6);
constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8);
constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9);
constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10);
constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11);
constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12);
constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13);
constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14);
constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15);
constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16);
constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100);
constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101);
constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION =
ToServerErrorCode(102);
constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104);
constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105);
constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106);
constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108);
constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109);
constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110);
constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111);
constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112);
constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113);
constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114);
constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115);
constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116);
constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117);
constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118);
constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119);
constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120);
constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121);
constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122);
// db error code
constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);
constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7);
constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8);
constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9);
constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10);
constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11);
constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12);
constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13);
constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14);
// knowhere error code
constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);
constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2);
constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3);
constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4);
// knowhere error code
constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1);
constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2);
constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3);
constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4);
// Snapshot error code
constexpr ErrorCode SS_ERROR = ToSSErrorCode(1);
constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2);
constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3);
constexpr ErrorCode SS_INVALID_CONTEXT_ERROR = ToSSErrorCode(4);
constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5);
constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6);
constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7);
constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8);
constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9);
constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10);
constexpr ErrorCode SS_NOT_COMMITTED = ToSSErrorCode(11);
constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12);
constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13);
} // namespace milvus

35
internal/core/src/utils/Json.h
View File

@ -1,35 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "nlohmann/json.hpp"
namespace milvus {
using json = nlohmann::json;
#define JSON_NULL_CHECK(json) \
do { \
if (json.empty()) { \
return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \
} \
} while (false)
#define JSON_OBJECT_CHECK(json) \
do { \
if (!json.is_object()) { \
return Status{SERVER_INVALID_ARGUMENT, \
"Json is not a json object"}; \
} \
} while (false)
} // namespace milvus

127
internal/core/src/utils/Status.cpp
View File

@ -1,127 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/Status.h"
#include "memory"
#include <cstring>
namespace milvus {
constexpr int CODE_WIDTH = sizeof(StatusCode);
Status::Status(StatusCode code, const std::string_view msg) {
// 4 bytes store code
// 4 bytes store message length
// the left bytes store message string
auto length = static_cast<uint32_t>(msg.size());
// auto result = new char[length + sizeof(length) + CODE_WIDTH];
state_.resize(length + sizeof(length) + CODE_WIDTH);
std::memcpy(state_.data(), &code, CODE_WIDTH);
std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length));
memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length);
}
Status::~Status() {
}
Status::Status(const Status& s) {
CopyFrom(s);
}
Status::Status(Status&& s) noexcept {
MoveFrom(s);
}
Status&
Status::operator=(const Status& s) {
CopyFrom(s);
return *this;
}
Status&
Status::operator=(Status&& s) noexcept {
MoveFrom(s);
return *this;
}
void
Status::CopyFrom(const Status& s) {
state_.clear();
if (s.state_.empty()) {
return;
}
uint32_t length = 0;
memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length));
int buff_len = length + sizeof(length) + CODE_WIDTH;
state_.resize(buff_len);
memcpy(state_.data(), s.state_.data(), buff_len);
}
void
Status::MoveFrom(Status& s) {
state_ = s.state_;
s.state_.clear();
}
std::string
Status::message() const {
if (state_.empty()) {
return "OK";
}
std::string msg;
uint32_t length = 0;
memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length));
if (length > 0) {
msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length);
}
return msg;
}
std::string
Status::ToString() const {
if (state_.empty()) {
return "OK";
}
std::string result;
switch (code()) {
case DB_SUCCESS:
result = "OK ";
break;
case DB_ERROR:
result = "Error: ";
break;
case DB_META_TRANSACTION_FAILED:
result = "Database error: ";
break;
case DB_NOT_FOUND:
result = "Not found: ";
break;
case DB_ALREADY_EXIST:
result = "Already exist: ";
break;
case DB_INVALID_PATH:
result = "Invalid path: ";
break;
default:
result = "Error code(" + std::to_string(code()) + "): ";
break;
}
result += message();
return result;
}
} // namespace milvus

79
internal/core/src/utils/Status.h
View File

@ -1,79 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "Error.h"
#include <string>
namespace milvus {
class Status;
#define STATUS_CHECK(func) \
do { \
Status s = func; \
if (!s.ok()) { \
return s; \
} \
} while (false)
using StatusCode = ErrorCode;
class Status {
public:
Status(StatusCode code, const std::string_view msg);
Status() = default;
virtual ~Status();
Status(const Status& s);
Status(Status&& s) noexcept;
Status&
operator=(const Status& s);
Status&
operator=(Status&& s) noexcept;
static Status
OK() {
return Status();
}
bool
ok() const {
return state_.empty() || code() == 0;
}
StatusCode
code() const {
return (state_.empty()) ? 0 : *(StatusCode*)(state_.data());
}
std::string
message() const;
std::string
ToString() const;
private:
inline void
CopyFrom(const Status& s);
inline void
MoveFrom(Status& s);
private:
std::string state_;
}; // Status
} // namespace milvus

3
internal/core/unittest/bench/CMakeLists.txt
View File

@ -12,7 +12,7 @@
include_directories(${CMAKE_HOME_DIRECTORY}/src)
include_directories(${CMAKE_HOME_DIRECTORY}/unittest)
set(bench_srcs
set(bench_srcs
bench_naive.cpp
bench_search.cpp
)
@ -38,7 +38,6 @@ target_link_libraries(indexbuilder_bench
milvus_log
pthread
knowhere
milvus_utils
)
target_link_libraries(indexbuilder_bench benchmark_main)

6
internal/core/unittest/test_data_codec.cpp
View File

@ -21,7 +21,7 @@
#include "storage/IndexData.h"
#include "storage/Util.h"
#include "common/Consts.h"
#include "utils/Json.h"
#include "common/Json.h"
using namespace milvus;
@ -305,8 +305,8 @@ TEST(storage, InsertDataBinaryVector) {
TEST(storage, InsertDataFloat16Vector) {
std::vector<float16> data = {1, 2, 3, 4, 5, 6, 7, 8};
int DIM = 2;
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::VECTOR_FLOAT16, DIM);
auto field_data = milvus::storage::CreateFieldData(
storage::DataType::VECTOR_FLOAT16, DIM);
field_data->FillFieldData(data.data(), data.size() / DIM);
storage::InsertData insert_data(field_data);

74
internal/core/unittest/test_index_c_api.cpp
View File

@ -48,34 +48,34 @@ TEST(FloatVecIndex, All) {
{
status = CreateIndex(
dtype, type_params_str.c_str(), index_params_str.c_str(), &index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = BuildFloatVecIndex(index, NB * DIM, xb_data.data());
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = SerializeIndexToBinarySet(index, &binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = CreateIndex(dtype,
type_params_str.c_str(),
index_params_str.c_str(),
&copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = LoadIndexFromBinarySet(copy_index, binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{ DeleteBinarySet(binary_set); }
}
@ -107,34 +107,34 @@ TEST(BinaryVecIndex, All) {
{
status = CreateIndex(
dtype, type_params_str.c_str(), index_params_str.c_str(), &index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = BuildBinaryVecIndex(index, NB * DIM / 8, xb_data.data());
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = SerializeIndexToBinarySet(index, &binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = CreateIndex(dtype,
type_params_str.c_str(),
index_params_str.c_str(),
&copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = LoadIndexFromBinarySet(copy_index, binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{ DeleteBinarySet(binary_set); }
}
@ -166,40 +166,40 @@ TEST(CBoolIndexTest, All) {
type_params_str.c_str(),
index_params_str.c_str(),
&index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = BuildScalarIndex(
index, half_ds->GetRows(), half_ds->GetTensor());
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = SerializeIndexToBinarySet(index, &binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = CreateIndex(dtype,
type_params_str.c_str(),
index_params_str.c_str(),
&copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = LoadIndexFromBinarySet(copy_index, binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{ DeleteBinarySet(binary_set); }
}
delete[](char*)(half_ds->GetTensor());
delete[] (char*)(half_ds->GetTensor());
}
// TODO: more scalar type.
@ -224,34 +224,34 @@ TEST(CInt64IndexTest, All) {
type_params_str.c_str(),
index_params_str.c_str(),
&index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = BuildScalarIndex(index, arr.size(), arr.data());
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = SerializeIndexToBinarySet(index, &binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = CreateIndex(dtype,
type_params_str.c_str(),
index_params_str.c_str(),
&copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = LoadIndexFromBinarySet(copy_index, binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{ DeleteBinarySet(binary_set); }
}
@ -283,39 +283,39 @@ TEST(CStringIndexTest, All) {
type_params_str.c_str(),
index_params_str.c_str(),
&index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = BuildScalarIndex(
index, (str_ds->GetRows()), (str_ds->GetTensor()));
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = SerializeIndexToBinarySet(index, &binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = CreateIndex(dtype,
type_params_str.c_str(),
index_params_str.c_str(),
&copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = LoadIndexFromBinarySet(copy_index, binary_set);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{
status = DeleteIndex(copy_index);
ASSERT_EQ(Success, status.error_code);
ASSERT_EQ(milvus::Success, status.error_code);
}
{ DeleteBinarySet(binary_set); }
}
delete[](char*)(str_ds->GetTensor());
delete[] (char*)(str_ds->GetTensor());
}
#endif

10
internal/core/unittest/test_indexing.cpp
View File

@ -16,7 +16,9 @@
#include <string>
#include <vector>
#include "common/EasyAssert.h"
#include "knowhere/comp/index_param.h"
#include "nlohmann/json.hpp"
#include "query/SearchBruteForce.h"
#include "segcore/Reduce.h"
#include "index/IndexFactory.h"
@ -55,7 +57,7 @@ generate_data(int N) {
}
} // namespace
Status
SegcoreError
merge_into(int64_t queries,
int64_t topk,
float* distances,
@ -90,7 +92,7 @@ merge_into(int64_t queries,
std::copy_n(buf_dis.data(), topk, src2_dis);
std::copy_n(buf_uids.data(), topk, src2_uids);
}
return Status::OK();
return SegcoreError::success();
}
/*
@ -173,7 +175,7 @@ TEST(Indexing, BinaryBruteForce) {
auto json = SearchResultToJson(sr);
std::cout << json.dump(2);
#ifdef __linux__
auto ref = json::parse(R"(
auto ref = nlohmann::json::parse(R"(
[
[
[ "1024->0.000000", "48942->0.642000", "18494->0.644000", "68225->0.644000", "93557->0.644000" ],
@ -190,7 +192,7 @@ TEST(Indexing, BinaryBruteForce) {
]
)");
#else // for mac
auto ref = json::parse(R"(
auto ref = nlohmann::json::parse(R"(
[
[
[ "1024->0.000000", "59169->0.645000", "98548->0.646000", "3356->0.646000", "90373->0.647000" ],

2
internal/core/unittest/test_parquet_c.cpp
View File

@ -17,11 +17,13 @@
#include <gtest/gtest.h>
#include <fstream>
#include "common/EasyAssert.h"
#include "storage/parquet_c.h"
#include "storage/PayloadReader.h"
#include "storage/PayloadWriter.h"
namespace wrapper = milvus::storage;
using ErrorCode = milvus::ErrorCodeEnum;
static void
WriteToFile(CBuffer cb) {

4
internal/core/unittest/test_query.cpp
View File

@ -22,6 +22,7 @@
#include "test_utils/AssertUtils.h"
#include "test_utils/DataGen.h"
using json = nlohmann::json;
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
@ -31,9 +32,6 @@ const int64_t ROW_COUNT = 100 * 1000;
}
TEST(Query, ShowExecutor) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto metric_type = knowhere::metric::L2;
auto node = std::make_unique<FloatVectorANNS>();
auto schema = std::make_shared<Schema>();

2
internal/core/unittest/test_tracer.cpp
View File

@ -15,7 +15,7 @@
#include <random>
#include "common/Tracer.h"
#include "exceptions/EasyAssert.h"
#include "common/EasyAssert.h"
using namespace milvus;
using namespace milvus::tracer;

19
internal/core/unittest/test_utils/DataGen.h
View File

@ -105,15 +105,13 @@ struct GeneratedData {
auto src_data = reinterpret_cast<const T*>(
target_field_data.vectors().binary_vector().data());
std::copy_n(src_data, len, ret.data());
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
// int len = raw_->num_rows() * field_meta.get_dim() * sizeof(float16);
int len = raw_->num_rows() * field_meta.get_dim();
ret.resize(len);
auto src_data =
reinterpret_cast<const T*>(target_field_data.vectors()
.float16_vector()
.data());
auto src_data = reinterpret_cast<const T*>(
target_field_data.vectors().float16_vector().data());
std::copy_n(src_data, len, ret.data());
} else {
PanicInfo("unsupported");
@ -269,7 +267,7 @@ DataGen(SchemaPtr schema,
insert_cols(data, N, field_meta);
break;
}
case DataType::VECTOR_FLOAT16: {
case DataType::VECTOR_FLOAT16: {
auto dim = field_meta.get_dim();
vector<float16> final(dim * N);
for (auto& x : final) {
@ -433,7 +431,8 @@ DataGenForJsonArray(SchemaPtr schema,
std::to_string(static_cast<double>(er())));
stringVec.push_back("\"" + std::to_string(er()) + "\"");
boolVec.push_back(i % 2 == 0 ? "true" : "false");
arrayVec.push_back(fmt::format("[{}, {}, {}]", i, i+1, i+2));
arrayVec.push_back(
fmt::format("[{}, {}, {}]", i, i + 1, i + 2));
}
auto str = R"({"int":[)" + join(intVec, ",") +
R"(],"double":[)" + join(doubleVec, ",") +
@ -604,7 +603,7 @@ SearchResultToVector(const SearchResult& sr) {
return result;
}
inline json
inline nlohmann::json
SearchResultToJson(const SearchResult& sr) {
int64_t num_queries = sr.total_nq_;
int64_t topk = sr.unity_topK_;
@ -618,7 +617,7 @@ SearchResultToJson(const SearchResult& sr) {
}
results.emplace_back(std::move(result));
}
return json{results};
return nlohmann::json{results};
};
inline storage::FieldDataPtr