milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support JSON data type (#23839) 

Signed-off-by: yah01 <yang.cen@zilliz.com>
pull/23970/head
yah01 2023-05-09 22:11:24 +08:00 committed by GitHub
parent f7159189fd
commit 62cabbfce2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
81 changed files with 4502 additions and 964 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
go.mod
View File

@ -28,7 +28,7 @@ require (
github.com/klauspost/compress v1.14.2
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230423035028-6708d3c12699
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230504042240-fb280598be89
github.com/minio/minio-go/v7 v7.0.17
github.com/opentracing/opentracing-go v1.2.0
github.com/panjf2000/ants/v2 v2.4.8
@ -133,7 +133,7 @@ require (
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.2.0
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/rs/xid v1.2.1 // indirect
@ -182,10 +182,15 @@ require (
sigs.k8s.io/yaml v1.2.0 // indirect
)
require github.com/cockroachdb/errors v1.2.4
require (
github.com/bytedance/sonic v1.8.8 // indirect
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/getsentry/raven-go v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1 // indirect

4
go.sum
View File

@ -499,8 +499,8 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyex
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230423035028-6708d3c12699 h1:Ka9007wF7Acht/JfEAq7rrqzSRT+c39M6+hT5hvrEvA=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230423035028-6708d3c12699/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230504042240-fb280598be89 h1:+PLcq1NIbn2HYbA/dkZEpkWwQwrgIYSnpyI5ORvowl4=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230504042240-fb280598be89/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/pulsar-client-go v0.6.8 h1:fZdZH73aPRszu2fazyeeahQEz34tyn1Pt9EkqJmV100=
github.com/milvus-io/pulsar-client-go v0.6.8/go.mod h1:oFIlYIk23tamkSLttw849qphmMIpHY8ztEBWDWJW+sc=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

8
internal/core/CMakeLists.txt
View File

@ -42,13 +42,7 @@ else ()
message(FATAL_ERROR "Unsupported platform!" )
endif ()
if (CMAKE_COMPILER_IS_GNUCC)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.99)
# ignore deprecated declarations for gcc>=12
# TODO: this workaround may removed when protobuf upgraded
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
endif ()
endif ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake" )
include( Utils )

1
internal/core/build-support/run_cpplint.py
View File

@ -40,6 +40,7 @@ _filters = '''
-build/c++11
-runtime/references
-build/include_order
-build/include_subdir
'''.split()

67
internal/core/src/common/CMakeLists.txt
View File

@ -1,48 +1,49 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License
milvus_add_pkg_config("milvus_common")
set(COMMON_SRC
Schema.cpp
SystemProperty.cpp
Slice.cpp
binary_set_c.cpp
init_c.cpp
Common.cpp
)
set(COMMON_SRC Schema.cpp SystemProperty.cpp Slice.cpp binary_set_c.cpp
init_c.cpp Common.cpp)
add_library(milvus_common SHARED ${COMMON_SRC})
if ( MSYS )
target_link_libraries(milvus_common
milvus_config
milvus_utils
milvus_log
yaml-cpp
boost_bitset_ext
arrow
parquet
)
if(MSYS)
target_link_libraries(
milvus_common
milvus_config
milvus_utils
milvus_log
yaml-cpp
boost_bitset_ext
arrow
parquet
simdjson
)
else()
target_link_libraries(milvus_common
milvus_config
milvus_utils
milvus_log
yaml-cpp
boost_bitset_ext
arrow
parquet
arrow_bundled
)
target_link_libraries(
milvus_common
milvus_config
milvus_utils
milvus_log
yaml-cpp
boost_bitset_ext
arrow
parquet
arrow_bundled
simdjson
)
endif()
install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}")

170
internal/core/src/common/Column.h Normal file
View File

@ -0,0 +1,170 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include <sys/mman.h>
#include <cstddef>
#include <ostream>
#include <string_view>
#include <type_traits>
#include <vector>
#include <string>
#include <utility>
#include "common/FieldMeta.h"
#include "common/Span.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "exceptions/EasyAssert.h"
#include "log/Log.h"
#include "nlohmann/json.hpp"
#include "storage/FieldData.h"
#include "storage/FieldDataInterface.h"
#include "storage/Util.h"
namespace milvus::segcore {
struct Entry {
char* data;
uint32_t length;
};
class ColumnBase {
public:
ColumnBase() = default;
virtual ~ColumnBase() {
if (data_ != nullptr && data_ != MAP_FAILED) {
if (munmap(data_, size_)) {
AssertInfo(true, std::string("failed to unmap variable field, err=") + strerror(errno));
}
}
}
ColumnBase(ColumnBase&& column) noexcept : data_(column.data_), size_(column.size_) {
column.data_ = nullptr;
column.size_ = 0;
}
const char*
data() const {
return data_;
}
[[nodiscard]] size_t
size() const {
return size_;
}
virtual SpanBase
span() const = 0;
protected:
char* data_{nullptr};
uint64_t size_{0};
};
class Column : public ColumnBase {
public:
Column(int64_t segment_id, const FieldMeta& field_meta, const std::vector<storage::FieldDataPtr>& field_data) {
auto row_num = storage::GetTotalNumRowsForFieldDatas(field_data);
data_ = static_cast<char*>(CreateMap(segment_id, field_meta, field_data));
size_ = field_meta.get_sizeof() * row_num;
row_count_ = row_num;
}
Column(Column&& column) noexcept : ColumnBase(std::move(column)), row_count_(column.row_count_) {
column.row_count_ = 0;
}
~Column() override = default;
SpanBase
span() const override {
return SpanBase(data_, row_count_, size_ / row_count_);
}
private:
int64_t row_count_{};
};
template <typename T>
class VariableColumn : public ColumnBase {
public:
using ViewType = std::conditional_t<std::is_same_v<T, std::string>, std::string_view, T>;
VariableColumn(int64_t segment_id,
const FieldMeta& field_meta,
const std::vector<storage::FieldDataPtr>& field_data) {
auto row_num = storage::GetTotalNumRowsForFieldDatas(field_data);
indices_.reserve(row_num);
for (const auto& data : field_data) {
auto field_data = static_cast<storage::FieldData<T>*>(data.get());
for (int i = 0; i < field_data->get_num_rows(); i++) {
auto row_data = reinterpret_cast<const std::string*>(field_data->RawValue(i));
indices_.emplace_back(size_);
size_ += row_data->size();
}
}
data_ = static_cast<char*>(CreateMap(segment_id, field_meta, field_data));
construct_views();
}
VariableColumn(VariableColumn&& field) noexcept
: indices_(std::move(field.indices_)), views_(std::move(field.views_)) {
data_ = field.data();
size_ = field.size();
field.data_ = nullptr;
}
~VariableColumn() override = default;
SpanBase
span() const override {
return SpanBase(views_.data(), views_.size(), sizeof(ViewType));
}
[[nodiscard]] const std::vector<ViewType>&
views() const {
return views_;
}
ViewType
operator[](const int i) const {
return views_[i];
}
std::string_view
raw_at(const int i) const {
size_t len = (i == indices_.size() - 1) ? size_ - indices_.back() : indices_[i + 1] - indices_[i];
return std::string_view(data_ + indices_[i], len);
}
protected:
void
construct_views() {
views_.reserve(indices_.size());
for (size_t i = 0; i < indices_.size() - 1; i++) {
views_.emplace_back(data_ + indices_[i], indices_[i + 1] - indices_[i]);
}
views_.emplace_back(data_ + indices_.back(), size_ - indices_.back());
}
private:
std::vector<uint64_t> indices_{};
// Compatible with current Span type
std::vector<ViewType> views_{};
};
} // namespace milvus::segcore

68
internal/core/src/common/FieldMeta.h
View File

@ -75,6 +75,10 @@ datatype_name(DataType data_type) {
return "double";
case DataType::VARCHAR:
return "varChar";
case DataType::ARRAY:
return "array";
case DataType::JSON:
return "json";
case DataType::VECTOR_FLOAT:
return "vector_float";
case DataType::VECTOR_BINARY: {
@ -103,6 +107,30 @@ datatype_is_string(DataType datatype) {
}
}
inline bool
datatype_is_binary(DataType datatype) {
switch (datatype) {
case DataType::ARRAY:
case DataType::JSON:
return true;
default:
return false;
}
}
inline bool
datatype_is_variable(DataType datatype) {
switch (datatype) {
case DataType::VARCHAR:
case DataType::STRING:
case DataType::ARRAY:
case DataType::JSON:
return true;
default:
return false;
}
}
inline bool
datatype_is_integer(DataType datatype) {
switch (datatype) {
@ -138,49 +166,37 @@ class FieldMeta {
operator=(FieldMeta&&) = default;
FieldMeta(const FieldName& name, FieldId id, DataType type) : name_(name), id_(id), type_(type) {
Assert(!is_vector());
Assert(!datatype_is_vector(type_));
}
FieldMeta(const FieldName& name, FieldId id, DataType type, int64_t max_length)
: name_(name), id_(id), type_(type), string_info_(StringInfo{max_length}) {
Assert(is_string());
Assert(datatype_is_string(type_));
}
FieldMeta(
const FieldName& name, FieldId id, DataType type, int64_t dim, std::optional<knowhere::MetricType> metric_type)
: name_(name), id_(id), type_(type), vector_info_(VectorInfo{dim, metric_type}) {
Assert(is_vector());
}
bool
is_vector() const {
Assert(type_ != DataType::NONE);
return type_ == DataType::VECTOR_BINARY || type_ == DataType::VECTOR_FLOAT;
}
bool
is_string() const {
Assert(type_ != DataType::NONE);
return type_ == DataType::VARCHAR || type_ == DataType::STRING;
Assert(datatype_is_vector(type_));
}
int64_t
get_dim() const {
Assert(is_vector());
Assert(datatype_is_vector(type_));
Assert(vector_info_.has_value());
return vector_info_->dim_;
}
int64_t
get_max_len() const {
Assert(is_string());
Assert(datatype_is_string(type_));
Assert(string_info_.has_value());
return string_info_->max_length;
}
std::optional<knowhere::MetricType>
get_metric_type() const {
Assert(is_vector());
Assert(datatype_is_vector(type_));
Assert(vector_info_.has_value());
return vector_info_->metric_type_;
}
@ -200,12 +216,26 @@ class FieldMeta {
return type_;
}
int64_t
bool
is_vector() const {
return datatype_is_vector(type_);
}
bool
is_string() const {
return datatype_is_string(type_);
}
size_t
get_sizeof() const {
static const size_t ARRAY_SIZE = 128;
static const size_t JSON_SIZE = 512;
if (is_vector()) {
return datatype_sizeof(type_, get_dim());
} else if (is_string()) {
return string_info_->max_length;
} else if (datatype_is_variable(type_)) {
return type_ == DataType::ARRAY ? ARRAY_SIZE : JSON_SIZE;
} else {
return datatype_sizeof(type_);
}

133
internal/core/src/common/Json.h Normal file
View File

@ -0,0 +1,133 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <cstddef>
#include <cstdlib>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "exceptions/EasyAssert.h"
#include "simdjson.h"
namespace milvus {
using document = simdjson::ondemand::document;
template <typename T>
using value_result = simdjson::simdjson_result<T>;
class Json {
public:
Json() = default;
explicit Json(simdjson::padded_string data) : own_data_(std::move(data)) {
data_ = own_data_.value();
}
Json(const char* data, size_t len, size_t cap) : data_(data, len) {
AssertInfo(len + simdjson::SIMDJSON_PADDING <= cap, "create json without enough memory size for SIMD, len=" +
std::to_string(len) + ", cap=" + std::to_string(cap));
}
// WARN: this is used for fast non-copy construction,
// MUST make sure that the data points to a memory that
// with size at least len + SIMDJSON_PADDING
Json(const char* data, size_t len) : data_(data, len) {
}
Json(const Json& json) {
if (json.own_data_.has_value()) {
own_data_ = simdjson::padded_string(json.own_data_.value().data(), json.own_data_.value().length());
data_ = own_data_.value();
} else {
data_ = json.data_;
}
}
Json(Json&& json) noexcept {
if (json.own_data_.has_value()) {
own_data_ = std::move(json.own_data_);
data_ = own_data_.value();
} else {
data_ = json.data_;
}
}
Json&
operator=(const Json& json) {
if (json.own_data_.has_value()) {
own_data_ = simdjson::padded_string(json.own_data_.value().data(), json.own_data_.value().length());
data_ = own_data_.value();
} else {
data_ = json.data_;
}
return *this;
}
operator std::string_view() const {
return data_;
}
document
doc() const {
thread_local simdjson::ondemand::parser parser;
// it's always safe to add the padding,
// as we have allocated the memory with this padding
document doc;
auto err = parser.iterate(data_, data_.size() + simdjson::SIMDJSON_PADDING).get(doc);
AssertInfo(err == simdjson::SUCCESS,
std::string("failed to parse the json ") + data_.data() + ": " + simdjson::error_message(err));
return doc;
}
bool
exist(std::vector<std::string> nested_path) const {
std::for_each(nested_path.begin(), nested_path.end(), [](std::string& key) {
boost::replace_all(key, "~", "~0");
boost::replace_all(key, "/", "~1");
});
auto pointer = "/" + boost::algorithm::join(nested_path, "/");
return doc().at_pointer(pointer).error() == simdjson::SUCCESS;
}
template <typename T>
value_result<T>
at(std::vector<std::string> nested_path) const {
std::for_each(nested_path.begin(), nested_path.end(), [](std::string& key) {
boost::replace_all(key, "~", "~0");
boost::replace_all(key, "/", "~1");
});
auto pointer = "/" + boost::algorithm::join(nested_path, "/");
return doc().at_pointer(pointer).get<T>();
}
std::string_view
data() const {
return data_;
}
private:
std::optional<simdjson::padded_string>
own_data_{}; // this could be empty, then the Json will be just s view on bytes
simdjson::padded_string_view data_{};
};
} // namespace milvus

2
internal/core/src/common/Types.h
View File

@ -58,6 +58,8 @@ enum class DataType {
STRING = 20,
VARCHAR = 21,
ARRAY = 22,
JSON = 23,
VECTOR_BINARY = 100,
VECTOR_FLOAT = 101,

28
internal/core/src/common/Utils.h
View File

@ -12,16 +12,28 @@
#pragma once
#include <google/protobuf/text_format.h>
#include <simdjson.h>
#include <algorithm>
#include <cstring>
#include <memory>
#include <string>
#include <fcntl.h>
#include <sys/mman.h>
#include "common/Consts.h"
#include "common/FieldMeta.h"
#include "common/Types.h"
#include "config/ConfigChunkManager.h"
#include "exceptions/EasyAssert.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
namespace milvus {
#define FIELD_DATA(data_array, type) (data_array->scalars().type##_data().data())
#define VEC_FIELD_DATA(data_array, type) (data_array->vectors().type##_vector().data())
inline DatasetPtr
GenDataset(const int64_t nb, const int64_t dim, const void* xb) {
@ -54,8 +66,8 @@ GetDatasetDim(const DatasetPtr& dataset) {
}
inline bool
PrefixMatch(const std::string& str, const std::string& prefix) {
auto ret = strncmp(str.c_str(), prefix.c_str(), prefix.length());
PrefixMatch(const std::string_view str, const std::string_view prefix) {
auto ret = strncmp(str.data(), prefix.data(), prefix.length());
if (ret != 0) {
return false;
}
@ -64,24 +76,16 @@ PrefixMatch(const std::string& str, const std::string& prefix) {
}
inline bool
PostfixMatch(const std::string& str, const std::string& postfix) {
PostfixMatch(const std::string_view str, const std::string_view postfix) {
if (postfix.length() > str.length()) {
return false;
}
int offset = str.length() - postfix.length();
auto ret = strncmp(str.c_str() + offset, postfix.c_str(), postfix.length());
auto ret = strncmp(str.data() + offset, postfix.data(), postfix.length());
if (ret != 0) {
return false;
}
//
// int i = postfix.length() - 1;
// int j = str.length() - 1;
// for (; i >= 0; i--, j--) {
// if (postfix[i] != str[j]) {
// return false;
// }
// }
return true;
}

5
internal/core/src/common/VectorTrait.h
View File

@ -16,7 +16,9 @@
#pragma once
#include "Types.h"
#include "Json.h"
#include <string>
#include <type_traits>
namespace milvus {
@ -49,7 +51,8 @@ template <typename T>
constexpr bool IsVector = std::is_base_of_v<VectorTrait, T>;
template <typename T>
constexpr bool IsScalar = std::is_fundamental_v<T> || std::is_same_v<T, std::string>;
constexpr bool IsScalar = std::is_fundamental_v<T> || std::is_same_v<T, std::string> || std::is_same_v<T, Json> ||
std::is_same_v<T, std::string_view>;
template <typename T, typename Enabled = void>
struct EmbeddedTypeImpl;

99
internal/core/src/pb/common.pb.cc
View File

@ -404,7 +404,7 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"\022\014\n\010NotExist\020\001\022\013\n\007Growing\020\002\022\n\n\006Sealed\020\003\022"
"\013\n\007Flushed\020\004\022\014\n\010Flushing\020\005\022\013\n\007Dropped\020\006\022"
"\r\n\tImporting\020\007*>\n\017PlaceholderType\022\010\n\004Non"
"e\020\000\022\020\n\014BinaryVector\020d\022\017\n\013FloatVector\020e*\263"
"e\020\000\022\020\n\014BinaryVector\020d\022\017\n\013FloatVector\020e*\357"
"\016\n\007MsgType\022\r\n\tUndefined\020\000\022\024\n\020CreateColle"
"ction\020d\022\022\n\016DropCollection\020e\022\021\n\rHasCollec"
"tion\020f\022\026\n\022DescribeCollection\020g\022\023\n\017ShowCo"
@ -451,51 +451,53 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"oup\020\244\r\022\026\n\021DropResourceGroup\020\245\r\022\027\n\022ListRe"
"sourceGroups\020\246\r\022\032\n\025DescribeResourceGroup"
"\020\247\r\022\021\n\014TransferNode\020\250\r\022\024\n\017TransferReplic"
"a\020\251\r*\"\n\007DslType\022\007\n\003Dsl\020\000\022\016\n\nBoolExprV1\020\001"
"*B\n\017CompactionState\022\021\n\rUndefiedState\020\000\022\r"
"\n\tExecuting\020\001\022\r\n\tCompleted\020\002*X\n\020Consiste"
"ncyLevel\022\n\n\006Strong\020\000\022\013\n\007Session\020\001\022\013\n\007Bou"
"nded\020\002\022\016\n\nEventually\020\003\022\016\n\nCustomized\020\004*\213"
"\001\n\013ImportState\022\021\n\rImportPending\020\000\022\020\n\014Imp"
"ortFailed\020\001\022\021\n\rImportStarted\020\002\022\023\n\017Import"
"Persisted\020\005\022\023\n\017ImportCompleted\020\006\022\032\n\026Impo"
"rtFailedAndCleaned\020\007*2\n\nObjectType\022\016\n\nCo"
"llection\020\000\022\n\n\006Global\020\001\022\010\n\004User\020\002*\365\007\n\017Obj"
"ectPrivilege\022\020\n\014PrivilegeAll\020\000\022\035\n\031Privil"
"egeCreateCollection\020\001\022\033\n\027PrivilegeDropCo"
"llection\020\002\022\037\n\033PrivilegeDescribeCollectio"
"n\020\003\022\034\n\030PrivilegeShowCollections\020\004\022\021\n\rPri"
"vilegeLoad\020\005\022\024\n\020PrivilegeRelease\020\006\022\027\n\023Pr"
"ivilegeCompaction\020\007\022\023\n\017PrivilegeInsert\020\010"
"\022\023\n\017PrivilegeDelete\020\t\022\032\n\026PrivilegeGetSta"
"tistics\020\n\022\030\n\024PrivilegeCreateIndex\020\013\022\030\n\024P"
"rivilegeIndexDetail\020\014\022\026\n\022PrivilegeDropIn"
"dex\020\r\022\023\n\017PrivilegeSearch\020\016\022\022\n\016PrivilegeF"
"lush\020\017\022\022\n\016PrivilegeQuery\020\020\022\030\n\024PrivilegeL"
"oadBalance\020\021\022\023\n\017PrivilegeImport\020\022\022\034\n\030Pri"
"vilegeCreateOwnership\020\023\022\027\n\023PrivilegeUpda"
"teUser\020\024\022\032\n\026PrivilegeDropOwnership\020\025\022\034\n\030"
"PrivilegeSelectOwnership\020\026\022\034\n\030PrivilegeM"
"anageOwnership\020\027\022\027\n\023PrivilegeSelectUser\020"
"\030\022 \n\034PrivilegeCreateResourceGroup\020\032\022\036\n\032P"
"rivilegeDropResourceGroup\020\033\022\"\n\036Privilege"
"DescribeResourceGroup\020\034\022\037\n\033PrivilegeList"
"ResourceGroups\020\035\022\031\n\025PrivilegeTransferNod"
"e\020\036\022\034\n\030PrivilegeTransferReplica\020\037\022\037\n\033Pri"
"vilegeGetLoadingProgress\020 \022\031\n\025PrivilegeG"
"etLoadState\020!\022\035\n\031PrivilegeRenameCollecti"
"on\020\"\022\033\n\027PrivilegeCreateDatabase\020#\022\031\n\025Pri"
"vilegeDropDatabase\020$\022\032\n\026PrivilegeListDat"
"abases\020%*S\n\tStateCode\022\020\n\014Initializing\020\000\022"
"\013\n\007Healthy\020\001\022\014\n\010Abnormal\020\002\022\013\n\007StandBy\020\003\022"
"\014\n\010Stopping\020\004*c\n\tLoadState\022\025\n\021LoadStateN"
"otExist\020\000\022\024\n\020LoadStateNotLoad\020\001\022\024\n\020LoadS"
"tateLoading\020\002\022\023\n\017LoadStateLoaded\020\003:^\n\021pr"
"ivilege_ext_obj\022\037.google.protobuf.Messag"
"eOptions\030\351\007 \001(\0132!.milvus.proto.common.Pr"
"ivilegeExtBU\n\016io.milvus.grpcB\013CommonProt"
"oP\001Z1github.com/milvus-io/milvus-proto/g"
"o-api/commonpb\240\001\001b\006proto3"
"a\020\251\r\022\023\n\016CreateDatabase\020\211\016\022\021\n\014DropDatabas"
"e\020\212\016\022\022\n\rListDatabases\020\213\016*\"\n\007DslType\022\007\n\003D"
"sl\020\000\022\016\n\nBoolExprV1\020\001*B\n\017CompactionState\022"
"\021\n\rUndefiedState\020\000\022\r\n\tExecuting\020\001\022\r\n\tCom"
"pleted\020\002*X\n\020ConsistencyLevel\022\n\n\006Strong\020\000"
"\022\013\n\007Session\020\001\022\013\n\007Bounded\020\002\022\016\n\nEventually"
"\020\003\022\016\n\nCustomized\020\004*\213\001\n\013ImportState\022\021\n\rIm"
"portPending\020\000\022\020\n\014ImportFailed\020\001\022\021\n\rImpor"
"tStarted\020\002\022\023\n\017ImportPersisted\020\005\022\023\n\017Impor"
"tCompleted\020\006\022\032\n\026ImportFailedAndCleaned\020\007"
"*2\n\nObjectType\022\016\n\nCollection\020\000\022\n\n\006Global"
"\020\001\022\010\n\004User\020\002*\365\007\n\017ObjectPrivilege\022\020\n\014Priv"
"ilegeAll\020\000\022\035\n\031PrivilegeCreateCollection\020"
"\001\022\033\n\027PrivilegeDropCollection\020\002\022\037\n\033Privil"
"egeDescribeCollection\020\003\022\034\n\030PrivilegeShow"
"Collections\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n\020Priv"
"ilegeRelease\020\006\022\027\n\023PrivilegeCompaction\020\007\022"
"\023\n\017PrivilegeInsert\020\010\022\023\n\017PrivilegeDelete\020"
"\t\022\032\n\026PrivilegeGetStatistics\020\n\022\030\n\024Privile"
"geCreateIndex\020\013\022\030\n\024PrivilegeIndexDetail\020"
"\014\022\026\n\022PrivilegeDropIndex\020\r\022\023\n\017PrivilegeSe"
"arch\020\016\022\022\n\016PrivilegeFlush\020\017\022\022\n\016PrivilegeQ"
"uery\020\020\022\030\n\024PrivilegeLoadBalance\020\021\022\023\n\017Priv"
"ilegeImport\020\022\022\034\n\030PrivilegeCreateOwnershi"
"p\020\023\022\027\n\023PrivilegeUpdateUser\020\024\022\032\n\026Privileg"
"eDropOwnership\020\025\022\034\n\030PrivilegeSelectOwner"
"ship\020\026\022\034\n\030PrivilegeManageOwnership\020\027\022\027\n\023"
"PrivilegeSelectUser\020\030\022 \n\034PrivilegeCreate"
"ResourceGroup\020\032\022\036\n\032PrivilegeDropResource"
"Group\020\033\022\"\n\036PrivilegeDescribeResourceGrou"
"p\020\034\022\037\n\033PrivilegeListResourceGroups\020\035\022\031\n\025"
"PrivilegeTransferNode\020\036\022\034\n\030PrivilegeTran"
"sferReplica\020\037\022\037\n\033PrivilegeGetLoadingProg"
"ress\020 \022\031\n\025PrivilegeGetLoadState\020!\022\035\n\031Pri"
"vilegeRenameCollection\020\"\022\033\n\027PrivilegeCre"
"ateDatabase\020#\022\031\n\025PrivilegeDropDatabase\020$"
"\022\032\n\026PrivilegeListDatabases\020%*S\n\tStateCod"
"e\022\020\n\014Initializing\020\000\022\013\n\007Healthy\020\001\022\014\n\010Abno"
"rmal\020\002\022\013\n\007StandBy\020\003\022\014\n\010Stopping\020\004*c\n\tLoa"
"dState\022\025\n\021LoadStateNotExist\020\000\022\024\n\020LoadSta"
"teNotLoad\020\001\022\024\n\020LoadStateLoading\020\002\022\023\n\017Loa"
"dStateLoaded\020\003:^\n\021privilege_ext_obj\022\037.go"
"ogle.protobuf.MessageOptions\030\351\007 \001(\0132!.mi"
"lvus.proto.common.PrivilegeExtBU\n\016io.mil"
"vus.grpcB\013CommonProtoP\001Z1github.com/milv"
"us-io/milvus-proto/go-api/commonpb\240\001\001b\006p"
"roto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_common_2eproto_deps[1] = {
&::descriptor_table_google_2fprotobuf_2fdescriptor_2eproto,
@ -516,7 +518,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_com
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_common_2eproto_once;
static bool descriptor_table_common_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_common_2eproto = {
&descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 6225,
&descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 6285,
&descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_sccs, descriptor_table_common_2eproto_deps, 11, 1,
schemas, file_default_instances, TableStruct_common_2eproto::offsets,
file_level_metadata_common_2eproto, 11, file_level_enum_descriptors_common_2eproto, file_level_service_descriptors_common_2eproto,
@ -746,6 +748,9 @@ bool MsgType_IsValid(int value) {
case 1703:
case 1704:
case 1705:
case 1801:
case 1802:
case 1803:
return true;
default:
return false;

5
internal/core/src/pb/common.pb.h
View File

@ -370,12 +370,15 @@ enum MsgType : int {
DescribeResourceGroup = 1703,
TransferNode = 1704,
TransferReplica = 1705,
CreateDatabase = 1801,
DropDatabase = 1802,
ListDatabases = 1803,
MsgType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
MsgType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max()
};
bool MsgType_IsValid(int value);
constexpr MsgType MsgType_MIN = Undefined;
constexpr MsgType MsgType_MAX = TransferReplica;
constexpr MsgType MsgType_MAX = ListDatabases;
constexpr int MsgType_ARRAYSIZE = MsgType_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* MsgType_descriptor();

251
internal/core/src/pb/plan.pb.cc
View File

@ -388,6 +388,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_plan_2eproto::offsets[] PROTOB
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnInfo, data_type_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnInfo, is_primary_key_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnInfo, is_autoid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnInfo, nested_path_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnExpr, _internal_metadata_),
~0u, // no _extensions_
@ -514,20 +515,20 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
{ 0, -1, sizeof(::milvus::proto::plan::GenericValue)},
{ 10, -1, sizeof(::milvus::proto::plan::QueryInfo)},
{ 19, -1, sizeof(::milvus::proto::plan::ColumnInfo)},
{ 28, -1, sizeof(::milvus::proto::plan::ColumnExpr)},
{ 34, -1, sizeof(::milvus::proto::plan::ValueExpr)},
{ 40, -1, sizeof(::milvus::proto::plan::UnaryRangeExpr)},
{ 48, -1, sizeof(::milvus::proto::plan::BinaryRangeExpr)},
{ 58, -1, sizeof(::milvus::proto::plan::CompareExpr)},
{ 66, -1, sizeof(::milvus::proto::plan::TermExpr)},
{ 73, -1, sizeof(::milvus::proto::plan::UnaryExpr)},
{ 80, -1, sizeof(::milvus::proto::plan::BinaryExpr)},
{ 88, -1, sizeof(::milvus::proto::plan::BinaryArithOp)},
{ 96, -1, sizeof(::milvus::proto::plan::BinaryArithExpr)},
{ 104, -1, sizeof(::milvus::proto::plan::BinaryArithOpEvalRangeExpr)},
{ 114, -1, sizeof(::milvus::proto::plan::Expr)},
{ 130, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 140, -1, sizeof(::milvus::proto::plan::PlanNode)},
{ 29, -1, sizeof(::milvus::proto::plan::ColumnExpr)},
{ 35, -1, sizeof(::milvus::proto::plan::ValueExpr)},
{ 41, -1, sizeof(::milvus::proto::plan::UnaryRangeExpr)},
{ 49, -1, sizeof(::milvus::proto::plan::BinaryRangeExpr)},
{ 59, -1, sizeof(::milvus::proto::plan::CompareExpr)},
{ 67, -1, sizeof(::milvus::proto::plan::TermExpr)},
{ 74, -1, sizeof(::milvus::proto::plan::UnaryExpr)},
{ 81, -1, sizeof(::milvus::proto::plan::BinaryExpr)},
{ 89, -1, sizeof(::milvus::proto::plan::BinaryArithOp)},
{ 97, -1, sizeof(::milvus::proto::plan::BinaryArithExpr)},
{ 105, -1, sizeof(::milvus::proto::plan::BinaryArithOpEvalRangeExpr)},
{ 115, -1, sizeof(::milvus::proto::plan::Expr)},
{ 131, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 141, -1, sizeof(::milvus::proto::plan::PlanNode)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@ -557,84 +558,85 @@ const char descriptor_table_protodef_plan_2eproto[] PROTOBUF_SECTION_VARIABLE(pr
"\001H\000\022\024\n\nstring_val\030\004 \001(\tH\000B\005\n\003val\"\\\n\tQuer"
"yInfo\022\014\n\004topk\030\001 \001(\003\022\023\n\013metric_type\030\003 \001(\t"
"\022\025\n\rsearch_params\030\004 \001(\t\022\025\n\rround_decimal"
"\030\005 \001(\003\"{\n\nColumnInfo\022\020\n\010field_id\030\001 \001(\003\0220"
"\n\tdata_type\030\002 \001(\0162\035.milvus.proto.schema."
"DataType\022\026\n\016is_primary_key\030\003 \001(\010\022\021\n\tis_a"
"utoID\030\004 \001(\010\"9\n\nColumnExpr\022+\n\004info\030\001 \001(\0132"
"\035.milvus.proto.plan.ColumnInfo\";\n\tValueE"
"xpr\022.\n\005value\030\001 \001(\0132\037.milvus.proto.plan.G"
"enericValue\"\233\001\n\016UnaryRangeExpr\0222\n\013column"
"_info\030\001 \001(\0132\035.milvus.proto.plan.ColumnIn"
"fo\022%\n\002op\030\002 \001(\0162\031.milvus.proto.plan.OpTyp"
"e\022.\n\005value\030\003 \001(\0132\037.milvus.proto.plan.Gen"
"ericValue\"\343\001\n\017BinaryRangeExpr\0222\n\013column_"
"info\030\001 \001(\0132\035.milvus.proto.plan.ColumnInf"
"o\022\027\n\017lower_inclusive\030\002 \001(\010\022\027\n\017upper_incl"
"usive\030\003 \001(\010\0224\n\013lower_value\030\004 \001(\0132\037.milvu"
"s.proto.plan.GenericValue\0224\n\013upper_value"
"\030\005 \001(\0132\037.milvus.proto.plan.GenericValue\""
"\247\001\n\013CompareExpr\0227\n\020left_column_info\030\001 \001("
"\0132\035.milvus.proto.plan.ColumnInfo\0228\n\021righ"
"t_column_info\030\002 \001(\0132\035.milvus.proto.plan."
"ColumnInfo\022%\n\002op\030\003 \001(\0162\031.milvus.proto.pl"
"an.OpType\"o\n\010TermExpr\0222\n\013column_info\030\001 \001"
"(\0132\035.milvus.proto.plan.ColumnInfo\022/\n\006val"
"ues\030\002 \003(\0132\037.milvus.proto.plan.GenericVal"
"ue\"\206\001\n\tUnaryExpr\0220\n\002op\030\001 \001(\0162$.milvus.pr"
"oto.plan.UnaryExpr.UnaryOp\022&\n\005child\030\002 \001("
"\0132\027.milvus.proto.plan.Expr\"\037\n\007UnaryOp\022\013\n"
"\007Invalid\020\000\022\007\n\003Not\020\001\"\307\001\n\nBinaryExpr\0222\n\002op"
"\030\001 \001(\0162&.milvus.proto.plan.BinaryExpr.Bi"
"naryOp\022%\n\004left\030\002 \001(\0132\027.milvus.proto.plan"
".Expr\022&\n\005right\030\003 \001(\0132\027.milvus.proto.plan"
".Expr\"6\n\010BinaryOp\022\013\n\007Invalid\020\000\022\016\n\nLogica"
"lAnd\020\001\022\r\n\tLogicalOr\020\002\"\255\001\n\rBinaryArithOp\022"
"2\n\013column_info\030\001 \001(\0132\035.milvus.proto.plan"
".ColumnInfo\0220\n\010arith_op\030\002 \001(\0162\036.milvus.p"
"roto.plan.ArithOpType\0226\n\rright_operand\030\003"
" \001(\0132\037.milvus.proto.plan.GenericValue\"\214\001"
"\n\017BinaryArithExpr\022%\n\004left\030\001 \001(\0132\027.milvus"
".proto.plan.Expr\022&\n\005right\030\002 \001(\0132\027.milvus"
".proto.plan.Expr\022*\n\002op\030\003 \001(\0162\036.milvus.pr"
"oto.plan.ArithOpType\"\221\002\n\032BinaryArithOpEv"
"alRangeExpr\0222\n\013column_info\030\001 \001(\0132\035.milvu"
"s.proto.plan.ColumnInfo\0220\n\010arith_op\030\002 \001("
"\0162\036.milvus.proto.plan.ArithOpType\0226\n\rrig"
"ht_operand\030\003 \001(\0132\037.milvus.proto.plan.Gen"
"ericValue\022%\n\002op\030\004 \001(\0162\031.milvus.proto.pla"
"n.OpType\022.\n\005value\030\005 \001(\0132\037.milvus.proto.p"
"lan.GenericValue\"\347\004\n\004Expr\0220\n\tterm_expr\030\001"
" \001(\0132\033.milvus.proto.plan.TermExprH\000\0222\n\nu"
"nary_expr\030\002 \001(\0132\034.milvus.proto.plan.Unar"
"yExprH\000\0224\n\013binary_expr\030\003 \001(\0132\035.milvus.pr"
"oto.plan.BinaryExprH\000\0226\n\014compare_expr\030\004 "
"\001(\0132\036.milvus.proto.plan.CompareExprH\000\022=\n"
"\020unary_range_expr\030\005 \001(\0132!.milvus.proto.p"
"lan.UnaryRangeExprH\000\022\?\n\021binary_range_exp"
"r\030\006 \001(\0132\".milvus.proto.plan.BinaryRangeE"
"xprH\000\022X\n\037binary_arith_op_eval_range_expr"
"\030\007 \001(\0132-.milvus.proto.plan.BinaryArithOp"
"EvalRangeExprH\000\022\?\n\021binary_arith_expr\030\010 \001"
"(\0132\".milvus.proto.plan.BinaryArithExprH\000"
"\0222\n\nvalue_expr\030\t \001(\0132\034.milvus.proto.plan"
".ValueExprH\000\0224\n\013column_expr\030\n \001(\0132\035.milv"
"us.proto.plan.ColumnExprH\000B\006\n\004expr\"\251\001\n\nV"
"ectorANNS\022\021\n\tis_binary\030\001 \001(\010\022\020\n\010field_id"
"\030\002 \001(\003\022+\n\npredicates\030\003 \001(\0132\027.milvus.prot"
"o.plan.Expr\0220\n\nquery_info\030\004 \001(\0132\034.milvus"
".proto.plan.QueryInfo\022\027\n\017placeholder_tag"
"\030\005 \001(\t\"\221\001\n\010PlanNode\0224\n\013vector_anns\030\001 \001(\013"
"2\035.milvus.proto.plan.VectorANNSH\000\022-\n\npre"
"dicates\030\002 \001(\0132\027.milvus.proto.plan.ExprH\000"
"\022\030\n\020output_field_ids\030\003 \003(\003B\006\n\004node*\272\001\n\006O"
"pType\022\013\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014G"
"reaterEqual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqual"
"\020\004\022\t\n\005Equal\020\005\022\014\n\010NotEqual\020\006\022\017\n\013PrefixMat"
"ch\020\007\022\020\n\014PostfixMatch\020\010\022\t\n\005Match\020\t\022\t\n\005Ran"
"ge\020\n\022\006\n\002In\020\013\022\t\n\005NotIn\020\014*G\n\013ArithOpType\022\013"
"\n\007Unknown\020\000\022\007\n\003Add\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007"
"\n\003Div\020\004\022\007\n\003Mod\020\005B3Z1github.com/milvus-io"
"/milvus/internal/proto/planpbb\006proto3"
"\030\005 \001(\003\"\220\001\n\nColumnInfo\022\020\n\010field_id\030\001 \001(\003\022"
"0\n\tdata_type\030\002 \001(\0162\035.milvus.proto.schema"
".DataType\022\026\n\016is_primary_key\030\003 \001(\010\022\021\n\tis_"
"autoID\030\004 \001(\010\022\023\n\013nested_path\030\005 \003(\t\"9\n\nCol"
"umnExpr\022+\n\004info\030\001 \001(\0132\035.milvus.proto.pla"
"n.ColumnInfo\";\n\tValueExpr\022.\n\005value\030\001 \001(\013"
"2\037.milvus.proto.plan.GenericValue\"\233\001\n\016Un"
"aryRangeExpr\0222\n\013column_info\030\001 \001(\0132\035.milv"
"us.proto.plan.ColumnInfo\022%\n\002op\030\002 \001(\0162\031.m"
"ilvus.proto.plan.OpType\022.\n\005value\030\003 \001(\0132\037"
".milvus.proto.plan.GenericValue\"\343\001\n\017Bina"
"ryRangeExpr\0222\n\013column_info\030\001 \001(\0132\035.milvu"
"s.proto.plan.ColumnInfo\022\027\n\017lower_inclusi"
"ve\030\002 \001(\010\022\027\n\017upper_inclusive\030\003 \001(\010\0224\n\013low"
"er_value\030\004 \001(\0132\037.milvus.proto.plan.Gener"
"icValue\0224\n\013upper_value\030\005 \001(\0132\037.milvus.pr"
"oto.plan.GenericValue\"\247\001\n\013CompareExpr\0227\n"
"\020left_column_info\030\001 \001(\0132\035.milvus.proto.p"
"lan.ColumnInfo\0228\n\021right_column_info\030\002 \001("
"\0132\035.milvus.proto.plan.ColumnInfo\022%\n\002op\030\003"
" \001(\0162\031.milvus.proto.plan.OpType\"o\n\010TermE"
"xpr\0222\n\013column_info\030\001 \001(\0132\035.milvus.proto."
"plan.ColumnInfo\022/\n\006values\030\002 \003(\0132\037.milvus"
".proto.plan.GenericValue\"\206\001\n\tUnaryExpr\0220"
"\n\002op\030\001 \001(\0162$.milvus.proto.plan.UnaryExpr"
".UnaryOp\022&\n\005child\030\002 \001(\0132\027.milvus.proto.p"
"lan.Expr\"\037\n\007UnaryOp\022\013\n\007Invalid\020\000\022\007\n\003Not\020"
"\001\"\307\001\n\nBinaryExpr\0222\n\002op\030\001 \001(\0162&.milvus.pr"
"oto.plan.BinaryExpr.BinaryOp\022%\n\004left\030\002 \001"
"(\0132\027.milvus.proto.plan.Expr\022&\n\005right\030\003 \001"
"(\0132\027.milvus.proto.plan.Expr\"6\n\010BinaryOp\022"
"\013\n\007Invalid\020\000\022\016\n\nLogicalAnd\020\001\022\r\n\tLogicalO"
"r\020\002\"\255\001\n\rBinaryArithOp\0222\n\013column_info\030\001 \001"
"(\0132\035.milvus.proto.plan.ColumnInfo\0220\n\010ari"
"th_op\030\002 \001(\0162\036.milvus.proto.plan.ArithOpT"
"ype\0226\n\rright_operand\030\003 \001(\0132\037.milvus.prot"
"o.plan.GenericValue\"\214\001\n\017BinaryArithExpr\022"
"%\n\004left\030\001 \001(\0132\027.milvus.proto.plan.Expr\022&"
"\n\005right\030\002 \001(\0132\027.milvus.proto.plan.Expr\022*"
"\n\002op\030\003 \001(\0162\036.milvus.proto.plan.ArithOpTy"
"pe\"\221\002\n\032BinaryArithOpEvalRangeExpr\0222\n\013col"
"umn_info\030\001 \001(\0132\035.milvus.proto.plan.Colum"
"nInfo\0220\n\010arith_op\030\002 \001(\0162\036.milvus.proto.p"
"lan.ArithOpType\0226\n\rright_operand\030\003 \001(\0132\037"
".milvus.proto.plan.GenericValue\022%\n\002op\030\004 "
"\001(\0162\031.milvus.proto.plan.OpType\022.\n\005value\030"
"\005 \001(\0132\037.milvus.proto.plan.GenericValue\"\347"
"\004\n\004Expr\0220\n\tterm_expr\030\001 \001(\0132\033.milvus.prot"
"o.plan.TermExprH\000\0222\n\nunary_expr\030\002 \001(\0132\034."
"milvus.proto.plan.UnaryExprH\000\0224\n\013binary_"
"expr\030\003 \001(\0132\035.milvus.proto.plan.BinaryExp"
"rH\000\0226\n\014compare_expr\030\004 \001(\0132\036.milvus.proto"
".plan.CompareExprH\000\022=\n\020unary_range_expr\030"
"\005 \001(\0132!.milvus.proto.plan.UnaryRangeExpr"
"H\000\022\?\n\021binary_range_expr\030\006 \001(\0132\".milvus.p"
"roto.plan.BinaryRangeExprH\000\022X\n\037binary_ar"
"ith_op_eval_range_expr\030\007 \001(\0132-.milvus.pr"
"oto.plan.BinaryArithOpEvalRangeExprH\000\022\?\n"
"\021binary_arith_expr\030\010 \001(\0132\".milvus.proto."
"plan.BinaryArithExprH\000\0222\n\nvalue_expr\030\t \001"
"(\0132\034.milvus.proto.plan.ValueExprH\000\0224\n\013co"
"lumn_expr\030\n \001(\0132\035.milvus.proto.plan.Colu"
"mnExprH\000B\006\n\004expr\"\251\001\n\nVectorANNS\022\021\n\tis_bi"
"nary\030\001 \001(\010\022\020\n\010field_id\030\002 \001(\003\022+\n\npredicat"
"es\030\003 \001(\0132\027.milvus.proto.plan.Expr\0220\n\nque"
"ry_info\030\004 \001(\0132\034.milvus.proto.plan.QueryI"
"nfo\022\027\n\017placeholder_tag\030\005 \001(\t\"\221\001\n\010PlanNod"
"e\0224\n\013vector_anns\030\001 \001(\0132\035.milvus.proto.pl"
"an.VectorANNSH\000\022-\n\npredicates\030\002 \001(\0132\027.mi"
"lvus.proto.plan.ExprH\000\022\030\n\020output_field_i"
"ds\030\003 \003(\003B\006\n\004node*\272\001\n\006OpType\022\013\n\007Invalid\020\000"
"\022\017\n\013GreaterThan\020\001\022\020\n\014GreaterEqual\020\002\022\014\n\010L"
"essThan\020\003\022\r\n\tLessEqual\020\004\022\t\n\005Equal\020\005\022\014\n\010N"
"otEqual\020\006\022\017\n\013PrefixMatch\020\007\022\020\n\014PostfixMat"
"ch\020\010\022\t\n\005Match\020\t\022\t\n\005Range\020\n\022\006\n\002In\020\013\022\t\n\005No"
"tIn\020\014*G\n\013ArithOpType\022\013\n\007Unknown\020\000\022\007\n\003Add"
"\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007\n\003Div\020\004\022\007\n\003Mod\020\005B3"
"Z1github.com/milvus-io/milvus/internal/p"
"roto/planpbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_plan_2eproto_deps[1] = {
&::descriptor_table_schema_2eproto,
@ -658,7 +660,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_pla
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_plan_2eproto_once;
static bool descriptor_table_plan_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_plan_2eproto = {
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 3357,
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 3379,
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_sccs, descriptor_table_plan_2eproto_deps, 14, 1,
schemas, file_default_instances, TableStruct_plan_2eproto::offsets,
file_level_metadata_plan_2eproto, 17, file_level_enum_descriptors_plan_2eproto, file_level_service_descriptors_plan_2eproto,
@ -1648,7 +1650,8 @@ ColumnInfo::ColumnInfo()
}
ColumnInfo::ColumnInfo(const ColumnInfo& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr) {
_internal_metadata_(nullptr),
nested_path_(from.nested_path_) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
::memcpy(&field_id_, &from.field_id_,
static_cast<size_t>(reinterpret_cast<char*>(&is_autoid_) -
@ -1657,6 +1660,7 @@ ColumnInfo::ColumnInfo(const ColumnInfo& from)
}
void ColumnInfo::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_ColumnInfo_plan_2eproto.base);
::memset(&field_id_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&is_autoid_) -
reinterpret_cast<char*>(&field_id_)) + sizeof(is_autoid_));
@ -1685,6 +1689,7 @@ void ColumnInfo::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
nested_path_.Clear();
::memset(&field_id_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&is_autoid_) -
reinterpret_cast<char*>(&field_id_)) + sizeof(is_autoid_));
@ -1728,6 +1733,18 @@ const char* ColumnInfo::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID:
CHK_(ptr);
} else goto handle_unusual;
continue;
// repeated string nested_path = 5;
case 5:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) {
ptr -= 1;
do {
ptr += 1;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_nested_path(), ptr, ctx, "milvus.proto.plan.ColumnInfo.nested_path");
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) break;
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -1811,6 +1828,22 @@ bool ColumnInfo::MergePartialFromCodedStream(
break;
}
// repeated string nested_path = 5;
case 5: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString(
input, this->add_nested_path()));
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->nested_path(this->nested_path_size() - 1).data(),
static_cast<int>(this->nested_path(this->nested_path_size() - 1).length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE,
"milvus.proto.plan.ColumnInfo.nested_path"));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -1859,6 +1892,16 @@ void ColumnInfo::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBool(4, this->is_autoid(), output);
}
// repeated string nested_path = 5;
for (int i = 0, n = this->nested_path_size(); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->nested_path(i).data(), static_cast<int>(this->nested_path(i).length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.plan.ColumnInfo.nested_path");
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString(
5, this->nested_path(i), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -1893,6 +1936,16 @@ void ColumnInfo::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(4, this->is_autoid(), target);
}
// repeated string nested_path = 5;
for (int i = 0, n = this->nested_path_size(); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->nested_path(i).data(), static_cast<int>(this->nested_path(i).length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.plan.ColumnInfo.nested_path");
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
WriteStringToArray(5, this->nested_path(i), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -1914,6 +1967,14 @@ size_t ColumnInfo::ByteSizeLong() const {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// repeated string nested_path = 5;
total_size += 1 *
::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->nested_path_size());
for (int i = 0, n = this->nested_path_size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->nested_path(i));
}
// int64 field_id = 1;
if (this->field_id() != 0) {
total_size += 1 +
@ -1964,6 +2025,7 @@ void ColumnInfo::MergeFrom(const ColumnInfo& from) {
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
nested_path_.MergeFrom(from.nested_path_);
if (from.field_id() != 0) {
set_field_id(from.field_id());
}
@ -1999,6 +2061,7 @@ bool ColumnInfo::IsInitialized() const {
void ColumnInfo::InternalSwap(ColumnInfo* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
nested_path_.InternalSwap(CastToBase(&other->nested_path_));
swap(field_id_, other->field_id_);
swap(data_type_, other->data_type_);
swap(is_primary_key_, other->is_primary_key_);

84
internal/core/src/pb/plan.pb.h
View File

@ -723,11 +723,29 @@ class ColumnInfo :
// accessors -------------------------------------------------------
enum : int {
kNestedPathFieldNumber = 5,
kFieldIdFieldNumber = 1,
kDataTypeFieldNumber = 2,
kIsPrimaryKeyFieldNumber = 3,
kIsAutoIDFieldNumber = 4,
};
// repeated string nested_path = 5;
int nested_path_size() const;
void clear_nested_path();
const std::string& nested_path(int index) const;
std::string* mutable_nested_path(int index);
void set_nested_path(int index, const std::string& value);
void set_nested_path(int index, std::string&& value);
void set_nested_path(int index, const char* value);
void set_nested_path(int index, const char* value, size_t size);
std::string* add_nested_path();
void add_nested_path(const std::string& value);
void add_nested_path(std::string&& value);
void add_nested_path(const char* value);
void add_nested_path(const char* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& nested_path() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_nested_path();
// int64 field_id = 1;
void clear_field_id();
::PROTOBUF_NAMESPACE_ID::int64 field_id() const;
@ -753,6 +771,7 @@ class ColumnInfo :
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> nested_path_;
::PROTOBUF_NAMESPACE_ID::int64 field_id_;
int data_type_;
bool is_primary_key_;
@ -3465,6 +3484,71 @@ inline void ColumnInfo::set_is_autoid(bool value) {
// @@protoc_insertion_point(field_set:milvus.proto.plan.ColumnInfo.is_autoID)
}
// repeated string nested_path = 5;
inline int ColumnInfo::nested_path_size() const {
return nested_path_.size();
}
inline void ColumnInfo::clear_nested_path() {
nested_path_.Clear();
}
inline const std::string& ColumnInfo::nested_path(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.ColumnInfo.nested_path)
return nested_path_.Get(index);
}
inline std::string* ColumnInfo::mutable_nested_path(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.plan.ColumnInfo.nested_path)
return nested_path_.Mutable(index);
}
inline void ColumnInfo::set_nested_path(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.plan.ColumnInfo.nested_path)
nested_path_.Mutable(index)->assign(value);
}
inline void ColumnInfo::set_nested_path(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.plan.ColumnInfo.nested_path)
nested_path_.Mutable(index)->assign(std::move(value));
}
inline void ColumnInfo::set_nested_path(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
nested_path_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.plan.ColumnInfo.nested_path)
}
inline void ColumnInfo::set_nested_path(int index, const char* value, size_t size) {
nested_path_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.plan.ColumnInfo.nested_path)
}
inline std::string* ColumnInfo::add_nested_path() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.plan.ColumnInfo.nested_path)
return nested_path_.Add();
}
inline void ColumnInfo::add_nested_path(const std::string& value) {
nested_path_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.plan.ColumnInfo.nested_path)
}
inline void ColumnInfo::add_nested_path(std::string&& value) {
nested_path_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.plan.ColumnInfo.nested_path)
}
inline void ColumnInfo::add_nested_path(const char* value) {
GOOGLE_DCHECK(value != nullptr);
nested_path_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.plan.ColumnInfo.nested_path)
}
inline void ColumnInfo::add_nested_path(const char* value, size_t size) {
nested_path_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.plan.ColumnInfo.nested_path)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
ColumnInfo::nested_path() const {
// @@protoc_insertion_point(field_list:milvus.proto.plan.ColumnInfo.nested_path)
return nested_path_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
ColumnInfo::mutable_nested_path() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.plan.ColumnInfo.nested_path)
return &nested_path_;
}
// -------------------------------------------------------------------
// ColumnExpr

1004
internal/core/src/pb/schema.pb.cc
View File

File diff suppressed because it is too large Load Diff

557
internal/core/src/pb/schema.pb.h
View File

@ -49,7 +49,7 @@ struct TableStruct_schema_2eproto {
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[14]
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[16]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
@ -59,6 +59,9 @@ extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table
namespace milvus {
namespace proto {
namespace schema {
class ArrayArray;
class ArrayArrayDefaultTypeInternal;
extern ArrayArrayDefaultTypeInternal _ArrayArray_default_instance_;
class BoolArray;
class BoolArrayDefaultTypeInternal;
extern BoolArrayDefaultTypeInternal _BoolArray_default_instance_;
@ -86,6 +89,9 @@ extern IDsDefaultTypeInternal _IDs_default_instance_;
class IntArray;
class IntArrayDefaultTypeInternal;
extern IntArrayDefaultTypeInternal _IntArray_default_instance_;
class JSONArray;
class JSONArrayDefaultTypeInternal;
extern JSONArrayDefaultTypeInternal _JSONArray_default_instance_;
class LongArray;
class LongArrayDefaultTypeInternal;
extern LongArrayDefaultTypeInternal _LongArray_default_instance_;
@ -105,6 +111,7 @@ extern VectorFieldDefaultTypeInternal _VectorField_default_instance_;
} // namespace proto
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template<> ::milvus::proto::schema::ArrayArray* Arena::CreateMaybeMessage<::milvus::proto::schema::ArrayArray>(Arena*);
template<> ::milvus::proto::schema::BoolArray* Arena::CreateMaybeMessage<::milvus::proto::schema::BoolArray>(Arena*);
template<> ::milvus::proto::schema::BytesArray* Arena::CreateMaybeMessage<::milvus::proto::schema::BytesArray>(Arena*);
template<> ::milvus::proto::schema::CollectionSchema* Arena::CreateMaybeMessage<::milvus::proto::schema::CollectionSchema>(Arena*);
@ -114,6 +121,7 @@ template<> ::milvus::proto::schema::FieldSchema* Arena::CreateMaybeMessage<::mil
template<> ::milvus::proto::schema::FloatArray* Arena::CreateMaybeMessage<::milvus::proto::schema::FloatArray>(Arena*);
template<> ::milvus::proto::schema::IDs* Arena::CreateMaybeMessage<::milvus::proto::schema::IDs>(Arena*);
template<> ::milvus::proto::schema::IntArray* Arena::CreateMaybeMessage<::milvus::proto::schema::IntArray>(Arena*);
template<> ::milvus::proto::schema::JSONArray* Arena::CreateMaybeMessage<::milvus::proto::schema::JSONArray>(Arena*);
template<> ::milvus::proto::schema::LongArray* Arena::CreateMaybeMessage<::milvus::proto::schema::LongArray>(Arena*);
template<> ::milvus::proto::schema::ScalarField* Arena::CreateMaybeMessage<::milvus::proto::schema::ScalarField>(Arena*);
template<> ::milvus::proto::schema::SearchResultData* Arena::CreateMaybeMessage<::milvus::proto::schema::SearchResultData>(Arena*);
@ -135,6 +143,8 @@ enum DataType : int {
Double = 11,
String = 20,
VarChar = 21,
Array = 22,
JSON = 23,
BinaryVector = 100,
FloatVector = 101,
DataType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
@ -310,6 +320,7 @@ class FieldSchema :
kIsPrimaryKeyFieldNumber = 3,
kAutoIDFieldNumber = 8,
kStateFieldNumber = 9,
kElementTypeFieldNumber = 10,
};
// repeated .milvus.proto.common.KeyValuePair type_params = 6;
int type_params_size() const;
@ -380,6 +391,11 @@ class FieldSchema :
::milvus::proto::schema::FieldState state() const;
void set_state(::milvus::proto::schema::FieldState value);
// .milvus.proto.schema.DataType element_type = 10;
void clear_element_type();
::milvus::proto::schema::DataType element_type() const;
void set_element_type(::milvus::proto::schema::DataType value);
// @@protoc_insertion_point(class_scope:milvus.proto.schema.FieldSchema)
private:
class _Internal;
@ -394,6 +410,7 @@ class FieldSchema :
bool is_primary_key_;
bool autoid_;
int state_;
int element_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_schema_2eproto;
};
@ -1545,6 +1562,293 @@ class StringArray :
};
// -------------------------------------------------------------------
class ArrayArray :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.schema.ArrayArray) */ {
public:
ArrayArray();
virtual ~ArrayArray();
ArrayArray(const ArrayArray& from);
ArrayArray(ArrayArray&& from) noexcept
: ArrayArray() {
*this = ::std::move(from);
}
inline ArrayArray& operator=(const ArrayArray& from) {
CopyFrom(from);
return *this;
}
inline ArrayArray& operator=(ArrayArray&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const ArrayArray& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const ArrayArray* internal_default_instance() {
return reinterpret_cast<const ArrayArray*>(
&_ArrayArray_default_instance_);
}
static constexpr int kIndexInFileMessages =
9;
friend void swap(ArrayArray& a, ArrayArray& b) {
a.Swap(&b);
}
inline void Swap(ArrayArray* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline ArrayArray* New() const final {
return CreateMaybeMessage<ArrayArray>(nullptr);
}
ArrayArray* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<ArrayArray>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const ArrayArray& from);
void MergeFrom(const ArrayArray& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(ArrayArray* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.schema.ArrayArray";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_schema_2eproto);
return ::descriptor_table_schema_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kDataFieldNumber = 1,
kElementTypeFieldNumber = 2,
};
// repeated .milvus.proto.schema.ScalarField data = 1;
int data_size() const;
void clear_data();
::milvus::proto::schema::ScalarField* mutable_data(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::schema::ScalarField >*
mutable_data();
const ::milvus::proto::schema::ScalarField& data(int index) const;
::milvus::proto::schema::ScalarField* add_data();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::schema::ScalarField >&
data() const;
// .milvus.proto.schema.DataType element_type = 2;
void clear_element_type();
::milvus::proto::schema::DataType element_type() const;
void set_element_type(::milvus::proto::schema::DataType value);
// @@protoc_insertion_point(class_scope:milvus.proto.schema.ArrayArray)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::schema::ScalarField > data_;
int element_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_schema_2eproto;
};
// -------------------------------------------------------------------
class JSONArray :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.schema.JSONArray) */ {
public:
JSONArray();
virtual ~JSONArray();
JSONArray(const JSONArray& from);
JSONArray(JSONArray&& from) noexcept
: JSONArray() {
*this = ::std::move(from);
}
inline JSONArray& operator=(const JSONArray& from) {
CopyFrom(from);
return *this;
}
inline JSONArray& operator=(JSONArray&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const JSONArray& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const JSONArray* internal_default_instance() {
return reinterpret_cast<const JSONArray*>(
&_JSONArray_default_instance_);
}
static constexpr int kIndexInFileMessages =
10;
friend void swap(JSONArray& a, JSONArray& b) {
a.Swap(&b);
}
inline void Swap(JSONArray* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline JSONArray* New() const final {
return CreateMaybeMessage<JSONArray>(nullptr);
}
JSONArray* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<JSONArray>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const JSONArray& from);
void MergeFrom(const JSONArray& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(JSONArray* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.schema.JSONArray";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_schema_2eproto);
return ::descriptor_table_schema_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kDataFieldNumber = 1,
};
// repeated bytes data = 1;
int data_size() const;
void clear_data();
const std::string& data(int index) const;
std::string* mutable_data(int index);
void set_data(int index, const std::string& value);
void set_data(int index, std::string&& value);
void set_data(int index, const char* value);
void set_data(int index, const void* value, size_t size);
std::string* add_data();
void add_data(const std::string& value);
void add_data(std::string&& value);
void add_data(const char* value);
void add_data(const void* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& data() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_data();
// @@protoc_insertion_point(class_scope:milvus.proto.schema.JSONArray)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> data_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_schema_2eproto;
};
// -------------------------------------------------------------------
class ScalarField :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.schema.ScalarField) */ {
public:
@ -1589,6 +1893,8 @@ class ScalarField :
kDoubleData = 5,
kStringData = 6,
kBytesData = 7,
kArrayData = 8,
kJsonData = 9,
DATA_NOT_SET = 0,
};
@ -1598,7 +1904,7 @@ class ScalarField :
&_ScalarField_default_instance_);
}
static constexpr int kIndexInFileMessages =
9;
11;
friend void swap(ScalarField& a, ScalarField& b) {
a.Swap(&b);
@ -1676,6 +1982,8 @@ class ScalarField :
kDoubleDataFieldNumber = 5,
kStringDataFieldNumber = 6,
kBytesDataFieldNumber = 7,
kArrayDataFieldNumber = 8,
kJsonDataFieldNumber = 9,
};
// .milvus.proto.schema.BoolArray bool_data = 1;
bool has_bool_data() const;
@ -1733,6 +2041,22 @@ class ScalarField :
::milvus::proto::schema::BytesArray* mutable_bytes_data();
void set_allocated_bytes_data(::milvus::proto::schema::BytesArray* bytes_data);
// .milvus.proto.schema.ArrayArray array_data = 8;
bool has_array_data() const;
void clear_array_data();
const ::milvus::proto::schema::ArrayArray& array_data() const;
::milvus::proto::schema::ArrayArray* release_array_data();
::milvus::proto::schema::ArrayArray* mutable_array_data();
void set_allocated_array_data(::milvus::proto::schema::ArrayArray* array_data);
// .milvus.proto.schema.JSONArray json_data = 9;
bool has_json_data() const;
void clear_json_data();
const ::milvus::proto::schema::JSONArray& json_data() const;
::milvus::proto::schema::JSONArray* release_json_data();
::milvus::proto::schema::JSONArray* mutable_json_data();
void set_allocated_json_data(::milvus::proto::schema::JSONArray* json_data);
void clear_data();
DataCase data_case() const;
// @@protoc_insertion_point(class_scope:milvus.proto.schema.ScalarField)
@ -1745,6 +2069,8 @@ class ScalarField :
void set_has_double_data();
void set_has_string_data();
void set_has_bytes_data();
void set_has_array_data();
void set_has_json_data();
inline bool has_data() const;
inline void clear_has_data();
@ -1759,6 +2085,8 @@ class ScalarField :
::milvus::proto::schema::DoubleArray* double_data_;
::milvus::proto::schema::StringArray* string_data_;
::milvus::proto::schema::BytesArray* bytes_data_;
::milvus::proto::schema::ArrayArray* array_data_;
::milvus::proto::schema::JSONArray* json_data_;
} data_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
::PROTOBUF_NAMESPACE_ID::uint32 _oneof_case_[1];
@ -1815,7 +2143,7 @@ class VectorField :
&_VectorField_default_instance_);
}
static constexpr int kIndexInFileMessages =
10;
12;
friend void swap(VectorField& a, VectorField& b) {
a.Swap(&b);
@ -1990,7 +2318,7 @@ class FieldData :
&_FieldData_default_instance_);
}
static constexpr int kIndexInFileMessages =
11;
13;
friend void swap(FieldData& a, FieldData& b) {
a.Swap(&b);
@ -2179,7 +2507,7 @@ class IDs :
&_IDs_default_instance_);
}
static constexpr int kIndexInFileMessages =
12;
14;
friend void swap(IDs& a, IDs& b) {
a.Swap(&b);
@ -2335,7 +2663,7 @@ class SearchResultData :
&_SearchResultData_default_instance_);
}
static constexpr int kIndexInFileMessages =
13;
15;
friend void swap(SearchResultData& a, SearchResultData& b) {
a.Swap(&b);
@ -2717,6 +3045,20 @@ inline void FieldSchema::set_state(::milvus::proto::schema::FieldState value) {
// @@protoc_insertion_point(field_set:milvus.proto.schema.FieldSchema.state)
}
// .milvus.proto.schema.DataType element_type = 10;
inline void FieldSchema::clear_element_type() {
element_type_ = 0;
}
inline ::milvus::proto::schema::DataType FieldSchema::element_type() const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.FieldSchema.element_type)
return static_cast< ::milvus::proto::schema::DataType >(element_type_);
}
inline void FieldSchema::set_element_type(::milvus::proto::schema::DataType value) {
element_type_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.schema.FieldSchema.element_type)
}
// -------------------------------------------------------------------
// CollectionSchema
@ -3177,6 +3519,123 @@ StringArray::mutable_data() {
// -------------------------------------------------------------------
// ArrayArray
// repeated .milvus.proto.schema.ScalarField data = 1;
inline int ArrayArray::data_size() const {
return data_.size();
}
inline void ArrayArray::clear_data() {
data_.Clear();
}
inline ::milvus::proto::schema::ScalarField* ArrayArray::mutable_data(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.schema.ArrayArray.data)
return data_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::schema::ScalarField >*
ArrayArray::mutable_data() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.schema.ArrayArray.data)
return &data_;
}
inline const ::milvus::proto::schema::ScalarField& ArrayArray::data(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.ArrayArray.data)
return data_.Get(index);
}
inline ::milvus::proto::schema::ScalarField* ArrayArray::add_data() {
// @@protoc_insertion_point(field_add:milvus.proto.schema.ArrayArray.data)
return data_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::schema::ScalarField >&
ArrayArray::data() const {
// @@protoc_insertion_point(field_list:milvus.proto.schema.ArrayArray.data)
return data_;
}
// .milvus.proto.schema.DataType element_type = 2;
inline void ArrayArray::clear_element_type() {
element_type_ = 0;
}
inline ::milvus::proto::schema::DataType ArrayArray::element_type() const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.ArrayArray.element_type)
return static_cast< ::milvus::proto::schema::DataType >(element_type_);
}
inline void ArrayArray::set_element_type(::milvus::proto::schema::DataType value) {
element_type_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.schema.ArrayArray.element_type)
}
// -------------------------------------------------------------------
// JSONArray
// repeated bytes data = 1;
inline int JSONArray::data_size() const {
return data_.size();
}
inline void JSONArray::clear_data() {
data_.Clear();
}
inline const std::string& JSONArray::data(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.JSONArray.data)
return data_.Get(index);
}
inline std::string* JSONArray::mutable_data(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.schema.JSONArray.data)
return data_.Mutable(index);
}
inline void JSONArray::set_data(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.schema.JSONArray.data)
data_.Mutable(index)->assign(value);
}
inline void JSONArray::set_data(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.schema.JSONArray.data)
data_.Mutable(index)->assign(std::move(value));
}
inline void JSONArray::set_data(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
data_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.schema.JSONArray.data)
}
inline void JSONArray::set_data(int index, const void* value, size_t size) {
data_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.schema.JSONArray.data)
}
inline std::string* JSONArray::add_data() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.schema.JSONArray.data)
return data_.Add();
}
inline void JSONArray::add_data(const std::string& value) {
data_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.schema.JSONArray.data)
}
inline void JSONArray::add_data(std::string&& value) {
data_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.schema.JSONArray.data)
}
inline void JSONArray::add_data(const char* value) {
GOOGLE_DCHECK(value != nullptr);
data_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.schema.JSONArray.data)
}
inline void JSONArray::add_data(const void* value, size_t size) {
data_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.schema.JSONArray.data)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
JSONArray::data() const {
// @@protoc_insertion_point(field_list:milvus.proto.schema.JSONArray.data)
return data_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
JSONArray::mutable_data() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.schema.JSONArray.data)
return &data_;
}
// -------------------------------------------------------------------
// ScalarField
// .milvus.proto.schema.BoolArray bool_data = 1;
@ -3466,6 +3925,88 @@ inline ::milvus::proto::schema::BytesArray* ScalarField::mutable_bytes_data() {
return data_.bytes_data_;
}
// .milvus.proto.schema.ArrayArray array_data = 8;
inline bool ScalarField::has_array_data() const {
return data_case() == kArrayData;
}
inline void ScalarField::set_has_array_data() {
_oneof_case_[0] = kArrayData;
}
inline void ScalarField::clear_array_data() {
if (has_array_data()) {
delete data_.array_data_;
clear_has_data();
}
}
inline ::milvus::proto::schema::ArrayArray* ScalarField::release_array_data() {
// @@protoc_insertion_point(field_release:milvus.proto.schema.ScalarField.array_data)
if (has_array_data()) {
clear_has_data();
::milvus::proto::schema::ArrayArray* temp = data_.array_data_;
data_.array_data_ = nullptr;
return temp;
} else {
return nullptr;
}
}
inline const ::milvus::proto::schema::ArrayArray& ScalarField::array_data() const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.ScalarField.array_data)
return has_array_data()
? *data_.array_data_
: *reinterpret_cast< ::milvus::proto::schema::ArrayArray*>(&::milvus::proto::schema::_ArrayArray_default_instance_);
}
inline ::milvus::proto::schema::ArrayArray* ScalarField::mutable_array_data() {
if (!has_array_data()) {
clear_data();
set_has_array_data();
data_.array_data_ = CreateMaybeMessage< ::milvus::proto::schema::ArrayArray >(
GetArenaNoVirtual());
}
// @@protoc_insertion_point(field_mutable:milvus.proto.schema.ScalarField.array_data)
return data_.array_data_;
}
// .milvus.proto.schema.JSONArray json_data = 9;
inline bool ScalarField::has_json_data() const {
return data_case() == kJsonData;
}
inline void ScalarField::set_has_json_data() {
_oneof_case_[0] = kJsonData;
}
inline void ScalarField::clear_json_data() {
if (has_json_data()) {
delete data_.json_data_;
clear_has_data();
}
}
inline ::milvus::proto::schema::JSONArray* ScalarField::release_json_data() {
// @@protoc_insertion_point(field_release:milvus.proto.schema.ScalarField.json_data)
if (has_json_data()) {
clear_has_data();
::milvus::proto::schema::JSONArray* temp = data_.json_data_;
data_.json_data_ = nullptr;
return temp;
} else {
return nullptr;
}
}
inline const ::milvus::proto::schema::JSONArray& ScalarField::json_data() const {
// @@protoc_insertion_point(field_get:milvus.proto.schema.ScalarField.json_data)
return has_json_data()
? *data_.json_data_
: *reinterpret_cast< ::milvus::proto::schema::JSONArray*>(&::milvus::proto::schema::_JSONArray_default_instance_);
}
inline ::milvus::proto::schema::JSONArray* ScalarField::mutable_json_data() {
if (!has_json_data()) {
clear_data();
set_has_json_data();
data_.json_data_ = CreateMaybeMessage< ::milvus::proto::schema::JSONArray >(
GetArenaNoVirtual());
}
// @@protoc_insertion_point(field_mutable:milvus.proto.schema.ScalarField.json_data)
return data_.json_data_;
}
inline bool ScalarField::has_data() const {
return data_case() != DATA_NOT_SET;
}
@ -4104,6 +4645,10 @@ SearchResultData::mutable_topks() {
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)

39
internal/core/src/query/Expr.h
View File

@ -25,6 +25,7 @@
#include <vector>
#include "common/Schema.h"
#include "common/Types.h"
#include "pb/plan.pb.h"
namespace milvus::query {
@ -33,6 +34,22 @@ using optype = proto::plan::OpType;
class ExprVisitor;
struct ColumnInfo {
FieldId field_id;
DataType data_type;
std::vector<std::string> nested_path;
explicit ColumnInfo(const proto::plan::ColumnInfo& column_info)
: field_id(column_info.field_id()),
data_type(static_cast<DataType>(column_info.data_type())),
nested_path(column_info.nested_path().begin(), column_info.nested_path().end()) {
}
ColumnInfo(FieldId field_id, DataType data_type, std::vector<std::string> nested_path = {})
: field_id(field_id), data_type(data_type), nested_path(std::move(nested_path)) {
}
};
// Base of all Exprs
struct Expr {
public:
@ -117,8 +134,8 @@ static const std::map<ArithOpType, std::string> mapping_arith_op_ = {
};
struct BinaryArithOpEvalRangeExpr : Expr {
const FieldId field_id_;
const DataType data_type_;
const ColumnInfo column_;
const proto::plan::GenericValue::ValCase val_case_;
const OpType op_type_;
const ArithOpType arith_op_;
@ -126,11 +143,11 @@ struct BinaryArithOpEvalRangeExpr : Expr {
// prevent accidential instantiation
BinaryArithOpEvalRangeExpr() = delete;
BinaryArithOpEvalRangeExpr(const FieldId field_id,
const DataType data_type,
BinaryArithOpEvalRangeExpr(ColumnInfo column,
const proto::plan::GenericValue::ValCase val_case,
const OpType op_type,
const ArithOpType arith_op)
: field_id_(field_id), data_type_(data_type), op_type_(op_type), arith_op_(arith_op) {
: column_(std::move(column)), val_case_(val_case), op_type_(op_type), arith_op_(arith_op) {
}
public:
@ -164,8 +181,8 @@ struct UnaryRangeExpr : Expr {
};
struct BinaryRangeExpr : Expr {
const FieldId field_id_;
const DataType data_type_;
const ColumnInfo column_;
const proto::plan::GenericValue::ValCase val_case_;
const bool lower_inclusive_;
const bool upper_inclusive_;
@ -173,12 +190,12 @@ struct BinaryRangeExpr : Expr {
// prevent accidential instantiation
BinaryRangeExpr() = delete;
BinaryRangeExpr(const FieldId field_id,
const DataType data_type,
BinaryRangeExpr(ColumnInfo column,
const proto::plan::GenericValue::ValCase val_case,
const bool lower_inclusive,
const bool upper_inclusive)
: field_id_(field_id),
data_type_(data_type),
: column_(std::move(column)),
val_case_(val_case),
lower_inclusive_(lower_inclusive),
upper_inclusive_(upper_inclusive) {
}

14
internal/core/src/query/ExprImpl.h
View File

@ -17,10 +17,12 @@
#pragma once
#include <tuple>
#include <utility>
#include <vector>
#include <boost/container/vector.hpp>
#include "Expr.h"
#include "pb/plan.pb.h"
namespace milvus::query {
@ -38,13 +40,13 @@ struct BinaryArithOpEvalRangeExprImpl : BinaryArithOpEvalRangeExpr {
const T right_operand_;
const T value_;
BinaryArithOpEvalRangeExprImpl(const FieldId field_id,
const DataType data_type,
BinaryArithOpEvalRangeExprImpl(ColumnInfo column,
const proto::plan::GenericValue::ValCase val_case,
const ArithOpType arith_op,
const T right_operand,
const OpType op_type,
const T value)
: BinaryArithOpEvalRangeExpr(field_id, data_type, op_type, arith_op),
: BinaryArithOpEvalRangeExpr(std::forward<ColumnInfo>(column), val_case, op_type, arith_op),
right_operand_(right_operand),
value_(value) {
}
@ -64,13 +66,13 @@ struct BinaryRangeExprImpl : BinaryRangeExpr {
const T lower_value_;
const T upper_value_;
BinaryRangeExprImpl(const FieldId field_id,
const DataType data_type,
BinaryRangeExprImpl(ColumnInfo column,
const proto::plan::GenericValue::ValCase val_case,
const bool lower_inclusive,
const bool upper_inclusive,
const T lower_value,
const T upper_value)
: BinaryRangeExpr(field_id, data_type, lower_inclusive, upper_inclusive),
: BinaryRangeExpr(std::forward<ColumnInfo>(column), val_case, lower_inclusive, upper_inclusive),
lower_value_(lower_value),
upper_value_(upper_value) {
}

16
internal/core/src/query/Parser.cpp
View File

@ -22,6 +22,8 @@
#include "Plan.h"
#include "generated/ExtractInfoPlanNodeVisitor.h"
#include "generated/VerifyPlanNodeVisitor.h"
#include "pb/plan.pb.h"
#include "query/Expr.h"
namespace milvus::query {
@ -92,7 +94,7 @@ Parser::ParseRangeNode(const Json& out_body) {
Assert(out_body.size() == 1);
auto out_iter = out_body.begin();
auto field_name = FieldName(out_iter.key());
auto body = out_iter.value();
auto& body = out_iter.value();
auto data_type = schema[field_name].get_data_type();
Assert(!datatype_is_vector(data_type));
@ -299,8 +301,9 @@ Parser::ParseRangeNodeImpl(const FieldName& field_name, const Json& body) {
}
return std::make_unique<BinaryArithOpEvalRangeExprImpl<T>>(
schema.get_field_id(field_name), schema[field_name].get_data_type(),
arith_op_mapping_.at(arith_op_name), right_operand, mapping_.at(op_name), value);
ColumnInfo(schema.get_field_id(field_name), schema[field_name].get_data_type()),
proto::plan::GenericValue::ValCase::VAL_NOT_SET, arith_op_mapping_.at(arith_op_name), right_operand,
mapping_.at(op_name), value);
}
if constexpr (std::is_same_v<T, bool>) {
@ -352,9 +355,10 @@ Parser::ParseRangeNodeImpl(const FieldName& field_name, const Json& body) {
}
}
AssertInfo(has_lower_value && has_upper_value, "illegal binary-range node");
return std::make_unique<BinaryRangeExprImpl<T>>(schema.get_field_id(field_name),
schema[field_name].get_data_type(), lower_inclusive,
upper_inclusive, lower_value, upper_value);
return std::make_unique<BinaryRangeExprImpl<T>>(
ColumnInfo(schema.get_field_id(field_name), schema[field_name].get_data_type()),
proto::plan::GenericValue::ValCase::VAL_NOT_SET, lower_inclusive, upper_inclusive, lower_value,
upper_value);
} else {
PanicInfo("illegal range node, too more or too few ops");
}

43
internal/core/src/query/PlanProto.cpp
View File

@ -11,13 +11,16 @@
#include <google/protobuf/text_format.h>
#include <cstdint>
#include <string>
#include "ExprImpl.h"
#include "PlanProto.h"
#include "common/VectorTrait.h"
#include "exceptions/EasyAssert.h"
#include "generated/ExtractInfoExprVisitor.h"
#include "generated/ExtractInfoPlanNodeVisitor.h"
#include "common/VectorTrait.h"
#include "pb/plan.pb.h"
#include "PlanProto.h"
namespace milvus::query {
namespace planpb = milvus::proto::plan;
@ -96,9 +99,9 @@ ExtractBinaryRangeExprImpl(FieldId field_id, DataType data_type, const planpb::B
static_assert(always_false<T>);
}
};
return std::make_unique<BinaryRangeExprImpl<T>>(field_id, data_type, expr_proto.lower_inclusive(),
expr_proto.upper_inclusive(), getValue(expr_proto.lower_value()),
getValue(expr_proto.upper_value()));
return std::make_unique<BinaryRangeExprImpl<T>>(
ColumnInfo(expr_proto.column_info()), expr_proto.lower_value().val_case(), expr_proto.lower_inclusive(),
expr_proto.upper_inclusive(), getValue(expr_proto.lower_value()), getValue(expr_proto.upper_value()));
}
template <typename T>
@ -122,8 +125,8 @@ ExtractBinaryArithOpEvalRangeExprImpl(FieldId field_id,
}
};
return std::make_unique<BinaryArithOpEvalRangeExprImpl<T>>(
field_id, data_type, static_cast<ArithOpType>(expr_proto.arith_op()), getValue(expr_proto.right_operand()),
static_cast<OpType>(expr_proto.op()), getValue(expr_proto.value()));
ColumnInfo(expr_proto.column_info()), expr_proto.value().val_case(), expr_proto.arith_op(),
getValue(expr_proto.right_operand()), expr_proto.op(), getValue(expr_proto.value()));
}
std::unique_ptr<VectorPlanNode>
@ -287,6 +290,21 @@ ProtoParser::ParseBinaryRangeExpr(const proto::plan::BinaryRangeExpr& expr_pb) {
case DataType::VARCHAR: {
return ExtractBinaryRangeExprImpl<std::string>(field_id, data_type, expr_pb);
}
case DataType::JSON: {
switch (expr_pb.lower_value().val_case()) {
case proto::plan::GenericValue::ValCase::kBoolVal:
return ExtractBinaryRangeExprImpl<bool>(field_id, data_type, expr_pb);
case proto::plan::GenericValue::ValCase::kInt64Val:
return ExtractBinaryRangeExprImpl<int64_t>(field_id, data_type, expr_pb);
case proto::plan::GenericValue::ValCase::kFloatVal:
return ExtractBinaryRangeExprImpl<double>(field_id, data_type, expr_pb);
case proto::plan::GenericValue::ValCase::kStringVal:
return ExtractBinaryRangeExprImpl<std::string>(field_id, data_type, expr_pb);
default:
PanicInfo("unknown data type in expression");
}
}
default: {
PanicInfo("unsupported data type");
}
@ -403,6 +421,17 @@ ProtoParser::ParseBinaryArithOpEvalRangeExpr(const proto::plan::BinaryArithOpEva
case DataType::DOUBLE: {
return ExtractBinaryArithOpEvalRangeExprImpl<double>(field_id, data_type, expr_pb);
}
case DataType::JSON: {
switch (expr_pb.value().val_case()) {
case proto::plan::GenericValue::ValCase::kInt64Val:
return ExtractBinaryArithOpEvalRangeExprImpl<int64_t>(field_id, data_type, expr_pb);
case proto::plan::GenericValue::ValCase::kFloatVal:
return ExtractBinaryArithOpEvalRangeExprImpl<double>(field_id, data_type, expr_pb);
default:
PanicInfo(std::string("unsupported expression data type ") +
std::to_string(expr_pb.value().val_case()));
}
}
default: {
PanicInfo("unsupported data type");
}

2
internal/core/src/query/SearchOnGrowing.cpp
View File

@ -44,7 +44,7 @@ FloatIndexSearch(const segcore::SegmentGrowingImpl& segment,
int current_chunk_id = 0;
if (indexing_record.is_in(vecfield_id)) {
auto max_indexed_id = indexing_record.get_finished_ack();
auto max_indexed_id = indexing_record.get_finished_ack(vecfield_id);
const auto& field_indexing = indexing_record.get_vec_field_indexing(vecfield_id);
auto search_params = field_indexing.get_search_params(info.topk_);
SearchInfo search_conf(info);

8
internal/core/src/query/SearchOnSealed.cpp
View File

@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <cmath>
#include <string>
#include "common/QueryInfo.h"
#include "query/SearchBruteForce.h"
@ -71,7 +72,7 @@ SearchOnSealedIndex(const Schema& schema,
void
SearchOnSealed(const Schema& schema,
const segcore::InsertRecord<true>& record,
const void* vec_data,
const SearchInfo& search_info,
const void* query_data,
int64_t num_queries,
@ -83,12 +84,9 @@ SearchOnSealed(const Schema& schema,
query::dataset::SearchDataset dataset{search_info.metric_type_, num_queries, search_info.topk_,
search_info.round_decimal_, field.get_dim(), query_data};
auto vec_data = record.get_field_data_base(field_id);
AssertInfo(vec_data->num_chunk() == 1, "num chunk not equal to 1 for sealed segment");
auto chunk_data = vec_data->get_chunk_data(0);
CheckBruteForceSearchParam(field, search_info);
auto sub_qr = BruteForceSearch(dataset, chunk_data, row_count, bitset);
auto sub_qr = BruteForceSearch(dataset, vec_data, row_count, bitset);
result.distances_ = std::move(sub_qr.mutable_distances());
result.seg_offsets_ = std::move(sub_qr.mutable_seg_offsets());

2
internal/core/src/query/SearchOnSealed.h
View File

@ -29,7 +29,7 @@ SearchOnSealedIndex(const Schema& schema,
void
SearchOnSealed(const Schema& schema,
const segcore::InsertRecord<true>& record,
const void* vec_data,
const SearchInfo& search_info,
const void* query_data,
int64_t num_queries,

13
internal/core/src/query/Utils.h
View File

@ -35,4 +35,17 @@ Match<std::string>(const std::string& str, const std::string& val, OpType op) {
PanicInfo("not supported");
}
}
template <>
inline bool
Match<std::string_view>(const std::string_view& str, const std::string& val, OpType op) {
switch (op) {
case OpType::PrefixMatch:
return PrefixMatch(str, val);
case OpType::PostfixMatch:
return PostfixMatch(str, val);
default:
PanicInfo("not supported");
}
}
} // namespace milvus::query

9
internal/core/src/query/generated/ExecExprVisitor.h
View File

@ -14,6 +14,7 @@
// DO NOT EDIT
#include <optional>
#include <boost/variant.hpp>
#include <type_traits>
#include <utility>
#include <deque>
#include "segcore/SegmentGrowingImpl.h"
@ -72,10 +73,18 @@ class ExecExprVisitor : public ExprVisitor {
auto
ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> BitsetType;
template <typename ExprValueType>
auto
ExecBinaryArithOpEvalRangeVisitorDispatcherJson(BinaryArithOpEvalRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRangeExpr& expr_raw) -> BitsetType;
template <typename ExprValueType>
auto
ExecBinaryRangeVisitorDispatcherJson(BinaryRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> BitsetType;

340
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -11,15 +11,24 @@
#include <deque>
#include <optional>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_set>
#include <utility>
#include <boost/variant.hpp>
#include "common/Json.h"
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "query/Relational.h"
#include "pb/plan.pb.h"
#include "query/ExprImpl.h"
#include "query/Utils.h"
#include "query/generated/ExecExprVisitor.h"
#include "segcore/SegmentGrowingImpl.h"
#include "query/Utils.h"
#include "query/Relational.h"
#include "simdjson/error.h"
#include "index/ScalarIndex.h"
namespace milvus::query {
// THIS CONTAINS EXTRA BODY FOR VISITOR
@ -158,9 +167,10 @@ ExecExprVisitor::ExecRangeVisitorImpl(FieldId field_id, IndexFunc index_func, El
auto num_chunk = upper_div(row_count_, size_per_chunk);
std::deque<BitsetType> results;
using Index = index::ScalarIndex<T>;
typedef std::conditional_t<std::is_same_v<T, std::string_view>, std::string, T> IndexInnerType;
using Index = index::ScalarIndex<IndexInnerType>;
for (auto chunk_id = 0; chunk_id < indexing_barrier; ++chunk_id) {
const Index& indexing = segment_.chunk_scalar_index<T>(field_id, chunk_id);
const Index& indexing = segment_.chunk_scalar_index<IndexInnerType>(field_id, chunk_id);
// NOTE: knowhere is not const-ready
// This is a dirty workaround
auto data = index_func(const_cast<Index*>(&indexing));
@ -175,7 +185,7 @@ ExecExprVisitor::ExecRangeVisitorImpl(FieldId field_id, IndexFunc index_func, El
for (int index = 0; index < this_size; ++index) {
result[index] = element_func(data[index]);
}
AssertInfo(result.size() == this_size, "");
results.emplace_back(std::move(result));
}
auto final_result = Assemble(results);
@ -215,9 +225,10 @@ ExecExprVisitor::ExecDataRangeVisitorImpl(FieldId field_id, IndexFunc index_func
// if sealed segment has loaded scalar index for this field, then index_barrier = 1 and data_barrier = 0
// in this case, sealed segment execute expr plan using scalar index
using Index = index::ScalarIndex<T>;
typedef std::conditional_t<std::is_same_v<T, std::string_view>, std::string, T> IndexInnerType;
using Index = index::ScalarIndex<IndexInnerType>;
for (auto chunk_id = data_barrier; chunk_id < indexing_barrier; ++chunk_id) {
auto& indexing = segment_.chunk_scalar_index<T>(field_id, chunk_id);
auto& indexing = segment_.chunk_scalar_index<IndexInnerType>(field_id, chunk_id);
auto this_size = const_cast<Index*>(&indexing)->Count();
BitsetType result(this_size);
for (int offset = 0; offset < this_size; ++offset) {
@ -236,10 +247,11 @@ ExecExprVisitor::ExecDataRangeVisitorImpl(FieldId field_id, IndexFunc index_func
template <typename T>
auto
ExecExprVisitor::ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<UnaryRangeExprImpl<T>&>(expr_raw);
using Index = index::ScalarIndex<T>;
typedef std::conditional_t<std::is_same_v<T, std::string_view>, std::string, T> IndexInnerType;
using Index = index::ScalarIndex<IndexInnerType>;
auto& expr = static_cast<UnaryRangeExprImpl<IndexInnerType>&>(expr_raw);
auto op = expr.op_type_;
auto val = expr.value_;
auto val = IndexInnerType(expr.value_);
switch (op) {
case OpType::Equal: {
auto index_func = [val](Index* index) { return index->In(1, &val); };
@ -300,6 +312,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
auto right_operand = expr.right_operand_;
auto op = expr.op_type_;
auto val = expr.value_;
auto& nested_path = expr.column_.nested_path;
switch (op) {
case OpType::Equal: {
@ -310,7 +323,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x + right_operand) == val;
};
auto elem_func = [val, right_operand](T x) { return ((x + right_operand) == val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Sub: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -318,7 +331,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x - right_operand) == val;
};
auto elem_func = [val, right_operand](T x) { return ((x - right_operand) == val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mul: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -326,7 +339,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x * right_operand) == val;
};
auto elem_func = [val, right_operand](T x) { return ((x * right_operand) == val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Div: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -334,7 +347,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x / right_operand) == val;
};
auto elem_func = [val, right_operand](T x) { return ((x / right_operand) == val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mod: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -344,7 +357,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
auto elem_func = [val, right_operand](T x) {
return (static_cast<T>(fmod(x, right_operand)) == val);
};
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
default: {
PanicInfo("unsupported arithmetic operation");
@ -359,7 +372,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x + right_operand) != val;
};
auto elem_func = [val, right_operand](T x) { return ((x + right_operand) != val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Sub: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -367,7 +380,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x - right_operand) != val;
};
auto elem_func = [val, right_operand](T x) { return ((x - right_operand) != val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mul: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -375,7 +388,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x * right_operand) != val;
};
auto elem_func = [val, right_operand](T x) { return ((x * right_operand) != val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Div: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -383,7 +396,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
return (x / right_operand) != val;
};
auto elem_func = [val, right_operand](T x) { return ((x / right_operand) != val); };
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mod: {
auto index_func = [val, right_operand](Index* index, size_t offset) {
@ -393,7 +406,7 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
auto elem_func = [val, right_operand](T x) {
return (static_cast<T>(fmod(x, right_operand)) != val);
};
return ExecDataRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecDataRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
default: {
PanicInfo("unsupported arithmetic operation");
@ -407,35 +420,197 @@ ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcher(BinaryArithOpEvalRa
}
#pragma clang diagnostic pop
template <typename ExprValueType>
auto
ExecExprVisitor::ExecBinaryArithOpEvalRangeVisitorDispatcherJson(BinaryArithOpEvalRangeExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<BinaryArithOpEvalRangeExprImpl<ExprValueType>&>(expr_raw);
using Index = index::ScalarIndex<milvus::Json>;
using GetType = std::conditional_t<std::is_same_v<ExprValueType, std::string>, std::string_view, ExprValueType>;
auto arith_op = expr.arith_op_;
auto right_operand = expr.right_operand_;
auto op = expr.op_type_;
auto val = expr.value_;
auto& nested_path = expr.column_.nested_path;
switch (op) {
case OpType::Equal: {
switch (arith_op) {
case ArithOpType::Add: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return !x.error() && ((x.value() + right_operand) == val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Sub: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return !x.error() && ((x.value() - right_operand) == val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mul: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return !x.error() && ((x.value() * right_operand) == val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Div: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return !x.error() && ((x.value() / right_operand) == val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mod: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return !x.error() && (static_cast<ExprValueType>(fmod(x.value(), right_operand)) == val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
default: {
PanicInfo("unsupported arithmetic operation");
}
}
}
case OpType::NotEqual: {
switch (arith_op) {
case ArithOpType::Add: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return x.error() || ((x.value() + right_operand) != val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Sub: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return x.error() || ((x.value() - right_operand) != val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mul: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return x.error() || ((x.value() * right_operand) != val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Div: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return x.error() || ((x.value() / right_operand) != val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
case ArithOpType::Mod: {
auto index_func = [val, right_operand](Index* index, size_t offset) { return false; };
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(nested_path);
return x.error() || (static_cast<ExprValueType>(fmod(x.value(), right_operand)) != val);
};
return ExecDataRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
default: {
PanicInfo("unsupported arithmetic operation");
}
}
}
default: {
PanicInfo("unsupported range node with arithmetic operation");
}
}
}
#pragma clang diagnostic push
#pragma ide diagnostic ignored "Simplify"
template <typename T>
auto
ExecExprVisitor::ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<BinaryRangeExprImpl<T>&>(expr_raw);
using Index = index::ScalarIndex<T>;
typedef std::conditional_t<std::is_same_v<T, std::string_view>, std::string, T> IndexInnerType;
using Index = index::ScalarIndex<IndexInnerType>;
auto& expr = static_cast<BinaryRangeExprImpl<IndexInnerType>&>(expr_raw);
bool lower_inclusive = expr.lower_inclusive_;
bool upper_inclusive = expr.upper_inclusive_;
T val1 = expr.lower_value_;
T val2 = expr.upper_value_;
IndexInnerType val1 = expr.lower_value_;
IndexInnerType val2 = expr.upper_value_;
auto index_func = [=](Index* index) { return index->Range(val1, lower_inclusive, val2, upper_inclusive); };
if (lower_inclusive && upper_inclusive) {
auto elem_func = [val1, val2](T x) { return (val1 <= x && x <= val2); };
return ExecRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
} else if (lower_inclusive && !upper_inclusive) {
auto elem_func = [val1, val2](T x) { return (val1 <= x && x < val2); };
return ExecRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
} else if (!lower_inclusive && upper_inclusive) {
auto elem_func = [val1, val2](T x) { return (val1 < x && x <= val2); };
return ExecRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
} else {
auto elem_func = [val1, val2](T x) { return (val1 < x && x < val2); };
return ExecRangeVisitorImpl<T>(expr.field_id_, index_func, elem_func);
return ExecRangeVisitorImpl<T>(expr.column_.field_id, index_func, elem_func);
}
}
#pragma clang diagnostic pop
template <typename ExprValueType>
auto
ExecExprVisitor::ExecBinaryRangeVisitorDispatcherJson(BinaryRangeExpr& expr_raw) -> BitsetType {
using Index = index::ScalarIndex<milvus::Json>;
using GetType = std::conditional_t<std::is_same_v<ExprValueType, std::string>, std::string_view, ExprValueType>;
auto& expr = static_cast<BinaryRangeExprImpl<ExprValueType>&>(expr_raw);
bool lower_inclusive = expr.lower_inclusive_;
bool upper_inclusive = expr.upper_inclusive_;
ExprValueType val1 = expr.lower_value_;
ExprValueType val2 = expr.upper_value_;
// no json index now
auto index_func = [=](Index* index) { return TargetBitmapPtr{}; };
if (lower_inclusive && upper_inclusive) {
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(expr.column_.nested_path);
auto value = x.value();
return !x.error() && (val1 <= value && value <= val2);
};
return ExecRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
} else if (lower_inclusive && !upper_inclusive) {
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(expr.column_.nested_path);
auto value = x.value();
return !x.error() && (val1 <= value && value < val2);
};
return ExecRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
} else if (!lower_inclusive && upper_inclusive) {
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(expr.column_.nested_path);
auto value = x.value();
return !x.error() && (val1 < value && value <= val2);
};
return ExecRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
} else {
auto elem_func = [&](const milvus::Json& json) {
auto x = json.template at<GetType>(expr.column_.nested_path);
auto value = x.value();
return !x.error() && (val1 < value && value < val2);
};
return ExecRangeVisitorImpl<milvus::Json>(expr.column_.field_id, index_func, elem_func);
}
}
void
ExecExprVisitor::visit(UnaryRangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_id_];
@ -472,7 +647,11 @@ ExecExprVisitor::visit(UnaryRangeExpr& expr) {
break;
}
case DataType::VARCHAR: {
res = ExecUnaryRangeVisitorDispatcher<std::string>(expr);
if (segment_.type() == SegmentType::Growing) {
res = ExecUnaryRangeVisitorDispatcher<std::string>(expr);
} else {
res = ExecUnaryRangeVisitorDispatcher<std::string_view>(expr);
}
break;
}
default:
@ -484,11 +663,11 @@ ExecExprVisitor::visit(UnaryRangeExpr& expr) {
void
ExecExprVisitor::visit(BinaryArithOpEvalRangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_id_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
auto& field_meta = segment_.get_schema()[expr.column_.field_id];
AssertInfo(expr.column_.data_type == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type");
BitsetType res;
switch (expr.data_type_) {
switch (expr.column_.data_type) {
case DataType::INT8: {
res = ExecBinaryArithOpEvalRangeVisitorDispatcher<int8_t>(expr);
break;
@ -513,6 +692,26 @@ ExecExprVisitor::visit(BinaryArithOpEvalRangeExpr& expr) {
res = ExecBinaryArithOpEvalRangeVisitorDispatcher<double>(expr);
break;
}
case DataType::JSON: {
switch (expr.val_case_) {
case proto::plan::GenericValue::ValCase::kBoolVal: {
res = ExecBinaryArithOpEvalRangeVisitorDispatcherJson<bool>(expr);
break;
}
case proto::plan::GenericValue::ValCase::kInt64Val: {
res = ExecBinaryArithOpEvalRangeVisitorDispatcherJson<int64_t>(expr);
break;
}
case proto::plan::GenericValue::ValCase::kFloatVal: {
res = ExecBinaryArithOpEvalRangeVisitorDispatcherJson<double>(expr);
break;
}
default: {
PanicInfo("unsupported value type {} in expression");
}
}
break;
}
default:
PanicInfo("unsupported");
}
@ -522,11 +721,11 @@ ExecExprVisitor::visit(BinaryArithOpEvalRangeExpr& expr) {
void
ExecExprVisitor::visit(BinaryRangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_id_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
auto& field_meta = segment_.get_schema()[expr.column_.field_id];
AssertInfo(expr.column_.data_type == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type");
BitsetType res;
switch (expr.data_type_) {
switch (expr.column_.data_type) {
case DataType::BOOL: {
res = ExecBinaryRangeVisitorDispatcher<bool>(expr);
break;
@ -556,7 +755,35 @@ ExecExprVisitor::visit(BinaryRangeExpr& expr) {
break;
}
case DataType::VARCHAR: {
res = ExecBinaryRangeVisitorDispatcher<std::string>(expr);
if (segment_.type() == SegmentType::Growing) {
res = ExecBinaryRangeVisitorDispatcher<std::string>(expr);
} else {
res = ExecBinaryRangeVisitorDispatcher<std::string_view>(expr);
}
break;
}
case DataType::JSON: {
switch (expr.val_case_) {
case proto::plan::GenericValue::ValCase::kBoolVal: {
res = ExecBinaryRangeVisitorDispatcherJson<bool>(expr);
break;
}
case proto::plan::GenericValue::ValCase::kInt64Val: {
res = ExecBinaryRangeVisitorDispatcherJson<int64_t>(expr);
break;
}
case proto::plan::GenericValue::ValCase::kFloatVal: {
res = ExecBinaryRangeVisitorDispatcherJson<double>(expr);
break;
}
case proto::plan::GenericValue::ValCase::kStringVal: {
res = ExecBinaryRangeVisitorDispatcherJson<std::string>(expr);
break;
}
default: {
PanicInfo("unsupported value type {} in expression");
}
}
break;
}
default:
@ -592,7 +819,8 @@ ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> BitsetTy
auto left_indexing_barrier = segment_.num_chunk_index(expr.left_field_id_);
auto left_data_barrier = segment_.num_chunk_data(expr.left_field_id_);
AssertInfo(std::max(left_data_barrier, left_indexing_barrier) == num_chunk,
"max(left_data_barrier, left_indexing_barrier) not equal to num_chunk");
"max(left_data_barrier, left_indexing_barrier) not equal to "
"num_chunk");
auto right_indexing_barrier = segment_.num_chunk_index(expr.right_field_id_);
auto right_data_barrier = segment_.num_chunk_data(expr.right_field_id_);
@ -676,8 +904,13 @@ ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> BitsetTy
}
case DataType::VARCHAR: {
if (chunk_id < data_barrier) {
auto chunk_data = segment_.chunk_data<std::string>(field_id, chunk_id).data();
return [chunk_data](int i) -> const number { return chunk_data[i]; };
if (segment_.type() == SegmentType::Growing) {
auto chunk_data = segment_.chunk_data<std::string>(field_id, chunk_id).data();
return [chunk_data](int i) -> const number { return chunk_data[i]; };
} else {
auto chunk_data = segment_.chunk_data<std::string_view>(field_id, chunk_id).data();
return [chunk_data](int i) -> const number { return std::string(chunk_data[i]); };
}
} else {
// for case, sealed segment has loaded index for scalar field instead of raw data
auto& indexing = segment_.chunk_scalar_index<std::string>(field_id, chunk_id);
@ -709,9 +942,11 @@ ExecExprVisitor::visit(CompareExpr& expr) {
auto& left_field_meta = schema[expr.left_field_id_];
auto& right_field_meta = schema[expr.right_field_id_];
AssertInfo(expr.left_data_type_ == left_field_meta.get_data_type(),
"[ExecExprVisitor]Left data type not equal to left field mata type");
"[ExecExprVisitor]Left data type not equal to left "
"field meta type");
AssertInfo(expr.right_data_type_ == right_field_meta.get_data_type(),
"[ExecExprVisitor]right data type not equal to right field mata type");
"[ExecExprVisitor]right data type not equal to right field "
"meta type");
BitsetType res;
switch (expr.op_type_) {
@ -808,12 +1043,19 @@ ExecExprVisitor::ExecTermVisitorImpl<std::string>(TermExpr& expr_raw) -> BitsetT
return ExecTermVisitorImplTemplate<std::string>(expr_raw);
}
template <>
auto
ExecExprVisitor::ExecTermVisitorImpl<std::string_view>(TermExpr& expr_raw) -> BitsetType {
return ExecTermVisitorImplTemplate<std::string_view>(expr_raw);
}
template <typename T>
auto
ExecExprVisitor::ExecTermVisitorImplTemplate(TermExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<TermExprImpl<T>&>(expr_raw);
using Index = index::ScalarIndex<T>;
const auto& terms = expr.terms_;
typedef std::conditional_t<std::is_same_v<T, std::string_view>, std::string, T> IndexInnerType;
using Index = index::ScalarIndex<IndexInnerType>;
auto& expr = static_cast<TermExprImpl<IndexInnerType>&>(expr_raw);
const std::vector<IndexInnerType> terms(expr.terms_.begin(), expr.terms_.end());
auto n = terms.size();
std::unordered_set<T> term_set(expr.terms_.begin(), expr.terms_.end());
@ -862,7 +1104,9 @@ void
ExecExprVisitor::visit(TermExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_id_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type ");
"[ExecExprVisitor]DataType of expr isn't field_meta "
"data type ");
BitsetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
@ -894,7 +1138,11 @@ ExecExprVisitor::visit(TermExpr& expr) {
break;
}
case DataType::VARCHAR: {
res = ExecTermVisitorImpl<std::string>(expr);
if (segment_.type() == SegmentType::Growing) {
res = ExecTermVisitorImpl<std::string>(expr);
} else {
res = ExecTermVisitorImpl<std::string_view>(expr);
}
break;
}
default:

1
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -99,6 +99,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
segment->mask_with_timestamps(*bitset_holder, timestamp_);
segment->mask_with_delete(*bitset_holder, active_count, timestamp_);
// if bitset_holder is all 1's, we got empty result
if (bitset_holder->all()) {
search_result_opt_ = empty_search_result(num_queries, node.search_info_);

4
internal/core/src/query/visitors/ExtractInfoExprVisitor.cpp
View File

@ -50,7 +50,7 @@ ExtractInfoExprVisitor::visit(UnaryRangeExpr& expr) {
void
ExtractInfoExprVisitor::visit(BinaryRangeExpr& expr) {
plan_info_.add_involved_field(expr.field_id_);
plan_info_.add_involved_field(expr.column_.field_id);
}
void
@ -61,7 +61,7 @@ ExtractInfoExprVisitor::visit(CompareExpr& expr) {
void
ExtractInfoExprVisitor::visit(BinaryArithOpEvalRangeExpr& expr) {
plan_info_.add_involved_field(expr.field_id_);
plan_info_.add_involved_field(expr.column_.field_id);
}
} // namespace milvus::query

18
internal/core/src/query/visitors/ShowExprVisitor.cpp
View File

@ -193,8 +193,8 @@ BinaryRangeExtract(const BinaryRangeExpr& expr_raw) {
auto expr = dynamic_cast<const BinaryRangeExprImpl<T>*>(&expr_raw);
AssertInfo(expr, "[ShowExprVisitor]BinaryRangeExpr cast to BinaryRangeExprImpl failed");
Json res{{"expr_type", "BinaryRange"},
{"field_id", expr->field_id_.get()},
{"data_type", datatype_name(expr->data_type_)},
{"field_id", expr->column_.field_id.get()},
{"data_type", datatype_name(expr->column_.data_type)},
{"lower_inclusive", expr->lower_inclusive_},
{"upper_inclusive", expr->upper_inclusive_},
{"lower_value", expr->lower_value_},
@ -205,8 +205,9 @@ BinaryRangeExtract(const BinaryRangeExpr& expr_raw) {
void
ShowExprVisitor::visit(BinaryRangeExpr& expr) {
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(datatype_is_vector(expr.data_type_) == false, "[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.data_type_) {
AssertInfo(datatype_is_vector(expr.column_.data_type) == false,
"[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.column_.data_type) {
case DataType::BOOL:
json_opt_ = BinaryRangeExtract<bool>(expr);
return;
@ -260,8 +261,8 @@ BinaryArithOpEvalRangeExtract(const BinaryArithOpEvalRangeExpr& expr_raw) {
AssertInfo(expr, "[ShowExprVisitor]BinaryArithOpEvalRangeExpr cast to BinaryArithOpEvalRangeExprImpl failed");
Json res{{"expr_type", "BinaryArithOpEvalRange"},
{"field_offset", expr->field_id_.get()},
{"data_type", datatype_name(expr->data_type_)},
{"field_offset", expr->column_.field_id.get()},
{"data_type", datatype_name(expr->column_.data_type)},
{"arith_op", ArithOpType_Name(static_cast<ArithOpType>(expr->arith_op_))},
{"right_operand", expr->right_operand_},
{"op", OpType_Name(static_cast<OpType>(expr->op_type_))},
@ -272,8 +273,9 @@ BinaryArithOpEvalRangeExtract(const BinaryArithOpEvalRangeExpr& expr_raw) {
void
ShowExprVisitor::visit(BinaryArithOpEvalRangeExpr& expr) {
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(datatype_is_vector(expr.data_type_) == false, "[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.data_type_) {
AssertInfo(datatype_is_vector(expr.column_.data_type) == false,
"[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.column_.data_type) {
case DataType::INT8:
json_opt_ = BinaryArithOpEvalRangeExtract<int8_t>(expr);
return;

39
internal/core/src/segcore/ConcurrentVector.cpp
View File

@ -10,6 +10,11 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "segcore/ConcurrentVector.h"
#include "common/FieldMeta.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "nlohmann/json.hpp"
#include "simdjson/simdjson.h"
namespace milvus::segcore {
@ -20,9 +25,9 @@ VectorBase::set_data_raw(ssize_t element_offset,
const FieldMeta& field_meta) {
if (field_meta.is_vector()) {
if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) {
return set_data_raw(element_offset, data->vectors().float_vector().data().data(), element_count);
return set_data_raw(element_offset, VEC_FIELD_DATA(data, float).data(), element_count);
} else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) {
return set_data_raw(element_offset, data->vectors().binary_vector().data(), element_count);
return set_data_raw(element_offset, VEC_FIELD_DATA(data, binary), element_count);
} else {
PanicInfo("unsupported");
}
@ -30,42 +35,50 @@ VectorBase::set_data_raw(ssize_t element_offset,
switch (field_meta.get_data_type()) {
case DataType::BOOL: {
return set_data_raw(element_offset, data->scalars().bool_data().data().data(), element_count);
return set_data_raw(element_offset, FIELD_DATA(data, bool).data(), element_count);
}
case DataType::INT8: {
auto src_data = data->scalars().int_data().data();
auto& src_data = FIELD_DATA(data, int);
std::vector<int8_t> data_raw(src_data.size());
std::copy_n(src_data.data(), src_data.size(), data_raw.data());
return set_data_raw(element_offset, data_raw.data(), element_count);
}
case DataType::INT16: {
auto src_data = data->scalars().int_data().data();
auto& src_data = FIELD_DATA(data, int);
std::vector<int16_t> data_raw(src_data.size());
std::copy_n(src_data.data(), src_data.size(), data_raw.data());
return set_data_raw(element_offset, data_raw.data(), element_count);
}
case DataType::INT32: {
return set_data_raw(element_offset, data->scalars().int_data().data().data(), element_count);
return set_data_raw(element_offset, FIELD_DATA(data, int).data(), element_count);
}
case DataType::INT64: {
return set_data_raw(element_offset, data->scalars().long_data().data().data(), element_count);
}
case DataType::FLOAT: {
return set_data_raw(element_offset, data->scalars().float_data().data().data(), element_count);
return set_data_raw(element_offset, FIELD_DATA(data, float).data(), element_count);
}
case DataType::DOUBLE: {
return set_data_raw(element_offset, data->scalars().double_data().data().data(), element_count);
return set_data_raw(element_offset, FIELD_DATA(data, double).data(), element_count);
}
case DataType::VARCHAR: {
auto begin = data->scalars().string_data().data().begin();
auto end = data->scalars().string_data().data().end();
std::vector<std::string> data_raw(begin, end);
auto& field_data = FIELD_DATA(data, string);
std::vector<std::string> data_raw(field_data.begin(), field_data.end());
return set_data_raw(element_offset, data_raw.data(), element_count);
}
case DataType::JSON: {
auto& json_data = FIELD_DATA(data, json);
std::vector<Json> data_raw{};
data_raw.reserve(json_data.size());
for (auto& json_bytes : json_data) {
data_raw.emplace_back(simdjson::padded_string(json_bytes));
}
return set_data_raw(element_offset, data_raw.data(), element_count);
}
default: {
PanicInfo("unsupported");
PanicInfo("unsupported datatype " + datatype_name(field_meta.get_data_type()));
}
}
}
} // namespace milvus::segcore

3
internal/core/src/segcore/ConcurrentVector.h
View File

@ -19,11 +19,13 @@
#include <mutex>
#include <shared_mutex>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
#include "common/FieldMeta.h"
#include "common/Json.h"
#include "common/Span.h"
#include "common/Types.h"
#include "common/Utils.h"
@ -305,6 +307,7 @@ class ConcurrentVectorImpl : public VectorBase {
AssertInfo(chunk_id < chunk_max_size, "chunk_id=" + std::to_string(chunk_id));
Chunk& chunk = chunks_[chunk_id];
auto ptr = chunk.data();
std::copy_n(source + source_offset * Dim, element_count * Dim, ptr + chunk_offset * Dim);
}

9
internal/core/src/segcore/FieldIndexing.h
View File

@ -131,6 +131,10 @@ class IndexingRecord {
continue;
}
}
if (field_meta.get_data_type() == DataType::ARRAY || field_meta.get_data_type() == DataType::JSON) {
// not supported yet
continue;
}
field_indexings_.try_emplace(field_id, CreateIndex(field_meta, segcore_config_));
}
@ -164,7 +168,10 @@ class IndexingRecord {
// concurrent
int64_t
get_finished_ack() const {
get_finished_ack(FieldId field_id) const {
if (field_indexings_.find(field_id) == field_indexings_.end()) {
return 0;
}
return finished_ack_.GetAck();
}

10
internal/core/src/segcore/InsertRecord.h
View File

@ -19,6 +19,7 @@
#include <utility>
#include "common/Schema.h"
#include "common/Types.h"
#include "segcore/AckResponder.h"
#include "segcore/ConcurrentVector.h"
#include "segcore/Record.h"
@ -208,6 +209,15 @@ struct InsertRecord {
this->append_field_data<std::string>(field_id, size_per_chunk);
break;
}
case DataType::JSON: {
this->append_field_data<Json>(field_id, size_per_chunk);
break;
}
// case DataType::ARRAY: {
// this->append_field_data<std::string>(field_id,
// size_per_chunk);
// break;
// }
default: {
PanicInfo("unsupported");
}

9
internal/core/src/segcore/SegmentGrowing.h
View File

@ -37,15 +37,16 @@ class SegmentGrowing : public SegmentInternalInterface {
const Timestamp* timestamps,
const InsertData* insert_data) = 0;
SegmentType
type() const override {
return Growing;
}
// virtual int64_t
// PreDelete(int64_t size) = 0;
// virtual Status
// Delete(int64_t reserved_offset, int64_t size, const int64_t* row_ids, const Timestamp* timestamps) = 0;
public:
virtual int64_t
get_deleted_count() const = 0;
};
using SegmentGrowingPtr = std::unique_ptr<SegmentGrowing>;

14
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -14,8 +14,11 @@
#include <queue>
#include <thread>
#include <boost/iterator/counting_iterator.hpp>
#include <type_traits>
#include "common/Consts.h"
#include "common/Types.h"
#include "nlohmann/json.hpp"
#include "query/PlanNode.h"
#include "query/SearchOnSealed.h"
#include "segcore/SegmentGrowingImpl.h"
@ -302,6 +305,11 @@ SegmentGrowingImpl::bulk_subscript(FieldId field_id, const int64_t* seg_offsets,
bulk_subscript_impl<std::string>(*vec_ptr, seg_offsets, count, output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
case DataType::JSON: {
FixedVector<std::string> output(count);
bulk_subscript_impl<Json, std::string>(*vec_ptr, seg_offsets, count, output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
default: {
PanicInfo("unsupported type");
}
@ -329,14 +337,14 @@ SegmentGrowingImpl::bulk_subscript_impl(int64_t element_sizeof,
}
}
template <typename T>
template <typename S, typename T>
void
SegmentGrowingImpl::bulk_subscript_impl(const VectorBase& vec_raw,
const int64_t* seg_offsets,
int64_t count,
void* output_raw) const {
static_assert(IsScalar<T>);
auto vec_ptr = dynamic_cast<const ConcurrentVector<T>*>(&vec_raw);
static_assert(IsScalar<S>);
auto vec_ptr = dynamic_cast<const ConcurrentVector<S>*>(&vec_raw);
AssertInfo(vec_ptr, "Pointer of vec_raw is nullptr");
auto& vec = *vec_ptr;
auto output = reinterpret_cast<T*>(output_raw);

5
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -29,6 +29,7 @@
#include "SealedIndexingRecord.h"
#include "SegmentGrowing.h"
#include "common/type_c.h"
#include "exceptions/EasyAssert.h"
#include "query/PlanNode.h"
#include "query/deprecated/GeneralQuery.h"
@ -101,7 +102,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
// return count of index that has index, i.e., [0, num_chunk_index) have built index
int64_t
num_chunk_index(FieldId field_id) const final {
return indexing_record_.get_finished_ack();
return indexing_record_.get_finished_ack(field_id);
}
// count of chunk that has raw data
@ -143,7 +144,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
get_active_count(Timestamp ts) const override;
// for scalar vectors
template <typename T>
template <typename S, typename T = S>
void
bulk_subscript_impl(const VectorBase& vec_raw, const int64_t* seg_offsets, int64_t count, void* output_raw) const;

1
internal/core/src/segcore/SegmentInterface.cpp
View File

@ -94,7 +94,6 @@ SegmentInternalInterface::Retrieve(const query::RetrievePlan* plan, Timestamp ti
auto data = reinterpret_cast<const int64_t*>(output.data());
auto obj = scalar_array->mutable_long_data();
obj->mutable_data()->Add(data, data + size);
fields_data->AddAllocated(data_array.release());
continue;
}

4
internal/core/src/segcore/SegmentInterface.h
View File

@ -27,6 +27,7 @@
#include "common/BitsetView.h"
#include "common/QueryResult.h"
#include "common/QueryInfo.h"
#include "common/type_c.h"
#include "query/Plan.h"
#include "query/PlanNode.h"
#include "pb/schema.pb.h"
@ -69,6 +70,9 @@ class SegmentInterface {
virtual int64_t
get_real_count() const = 0;
virtual SegmentType
type() const = 0;
virtual int64_t
PreDelete(int64_t size) = 0;

7
internal/core/src/segcore/SegmentSealed.h
View File

@ -15,9 +15,11 @@
#include <utility>
#include <vector>
#include "common/type_c.h"
#include "pb/segcore.pb.h"
#include "segcore/SegmentInterface.h"
#include "segcore/Types.h"
#include "storage/FieldData.h"
namespace milvus::segcore {
@ -33,6 +35,11 @@ class SegmentSealed : public SegmentInternalInterface {
DropFieldData(const FieldId field_id) = 0;
virtual void
LoadFieldData(FieldId field_id, const std::vector<storage::FieldDataPtr>& field_datas) = 0;
SegmentType
type() const override {
return Sealed;
}
};
using SegmentSealedPtr = std::unique_ptr<SegmentSealed>;

110
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -14,6 +14,9 @@
#include "Utils.h"
#include "common/Consts.h"
#include "common/FieldMeta.h"
#include "common/Types.h"
#include "common/Column.h"
#include "log/Log.h"
#include "query/ScalarIndex.h"
#include "query/SearchBruteForce.h"
#include "query/SearchOnSealed.h"
@ -159,8 +162,8 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& load_info) {
}
void
SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::FieldDataPtr>& field_datas) {
int64_t size = storage::GetTotalNumRowsForFieldDatas(field_datas);
SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::FieldDataPtr>& field_data) {
int64_t size = storage::GetTotalNumRowsForFieldDatas(field_data);
if (row_count_opt_.has_value()) {
AssertInfo(row_count_opt_.value() == size, "field (" + std::to_string(field_id.get()) +
") data has different row count (" + std::to_string(size) +
@ -173,7 +176,7 @@ SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::Fi
if (system_field_type == SystemFieldType::Timestamp) {
std::vector<Timestamp> timestamps(size);
int64_t offset = 0;
for (auto& data : field_datas) {
for (auto& data : field_data) {
int64_t row_count = data->get_num_rows();
std::copy_n(static_cast<const Timestamp*>(data->Data()), row_count, timestamps.data() + offset);
offset += row_count;
@ -188,7 +191,7 @@ SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::Fi
// use special index
std::unique_lock lck(mutex_);
AssertInfo(insert_record_.timestamps_.empty(), "already exists");
insert_record_.timestamps_.fill_chunk_data(field_datas);
insert_record_.timestamps_.fill_chunk_data(field_data);
insert_record_.timestamp_index_ = std::move(index);
AssertInfo(insert_record_.timestamps_.num_chunk() == 1, "num chunk not equal to 1 for sealed segment");
} else {
@ -196,7 +199,7 @@ SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::Fi
// write data under lock
std::unique_lock lck(mutex_);
AssertInfo(insert_record_.row_ids_.empty(), "already exists");
insert_record_.row_ids_.fill_chunk_data(field_datas);
insert_record_.row_ids_.fill_chunk_data(field_data);
AssertInfo(insert_record_.row_ids_.num_chunk() == 1, "num chunk not equal to 1 for sealed segment");
}
++system_ready_count_;
@ -204,20 +207,40 @@ SegmentSealedImpl::LoadFieldData(FieldId field_id, const std::vector<storage::Fi
// write data under lock
std::unique_lock lck(mutex_);
auto& field_meta = (*schema_)[field_id];
auto data_type = field_meta.get_data_type();
// Don't allow raw data and index exist at the same time
AssertInfo(!get_bit(index_ready_bitset_, field_id), "field data can't be loaded when indexing exists");
auto field_data = insert_record_.get_field_data_base(field_id);
AssertInfo(field_data->empty(), "already exists");
// insert data to insertRecord
field_data->fill_chunk_data(field_datas);
AssertInfo(field_data->num_chunk() == 1, "num chunk not equal to 1 for sealed segment");
size_t size = 0;
if (datatype_is_variable(data_type)) {
std::unique_ptr<ColumnBase> column{};
switch (data_type) {
case milvus::DataType::STRING:
case milvus::DataType::VARCHAR: {
column = std::make_unique<VariableColumn<std::string>>(get_segment_id(), field_meta, field_data);
break;
}
case milvus::DataType::JSON: {
column = std::make_unique<VariableColumn<Json>>(get_segment_id(), field_meta, field_data);
}
default: {
}
}
size = column->size();
variable_fields_.emplace(field_id, std::move(column));
} else {
auto column = Column(get_segment_id(), field_meta, field_data);
size = column.size();
fixed_fields_.emplace(field_id, std::move(column));
}
// set pks to offset
if (schema_->get_primary_field_id() == field_id) {
AssertInfo(field_id.get() != -1, "Primary key is -1");
AssertInfo(insert_record_.empty_pks(), "already exists");
insert_record_.insert_pks(field_datas);
insert_record_.insert_pks(field_data);
insert_record_.seal_pks();
}
@ -260,9 +283,7 @@ SegmentSealedImpl::num_chunk_index(FieldId field_id) const {
int64_t
SegmentSealedImpl::num_chunk_data(FieldId field_id) const {
auto field_data = insert_record_.get_field_data_base(field_id);
AssertInfo(field_data != nullptr, "null field data ptr");
return field_data->num_chunk();
return get_bit(field_data_ready_bitset_, field_id) ? 1 : 0;
}
int64_t
@ -282,6 +303,14 @@ SegmentSealedImpl::chunk_data_impl(FieldId field_id, int64_t chunk_id) const {
"Can't get bitset element at " + std::to_string(field_id.get()));
auto& field_meta = schema_->operator[](field_id);
auto element_sizeof = field_meta.get_sizeof();
if (auto it = fixed_fields_.find(field_id); it != fixed_fields_.end()) {
auto& field_data = it->second;
return field_data.span();
}
if (auto it = variable_fields_.find(field_id); it != variable_fields_.end()) {
auto& field = it->second;
return field->span();
}
auto field_data = insert_record_.get_field_data_base(field_id);
AssertInfo(field_data->num_chunk() == 1, "num chunk not equal to 1 for sealed segment");
return field_data->get_span_base(0);
@ -326,6 +355,7 @@ SegmentSealedImpl::mask_with_delete(BitsetType& bitset, int64_t ins_barrier, Tim
if (del_barrier == 0) {
return;
}
auto bitmap_holder = get_deleted_bitmap(del_barrier, ins_barrier, deleted_record_, insert_record_, timestamp);
if (!bitmap_holder || !bitmap_holder->bitmap_ptr) {
return;
@ -356,7 +386,8 @@ SegmentSealedImpl::vector_search(SearchInfo& search_info,
"Field Data is not loaded: " + std::to_string(field_id.get()));
AssertInfo(row_count_opt_.has_value(), "Can't get row count value");
auto row_count = row_count_opt_.value();
query::SearchOnSealed(*schema_, insert_record_, search_info, query_data, query_count, row_count, bitset,
auto& vec_data = fixed_fields_.at(field_id);
query::SearchOnSealed(*schema_, vec_data.data(), search_info, query_data, query_count, row_count, bitset,
output);
}
}
@ -464,6 +495,22 @@ SegmentSealedImpl::bulk_subscript_impl(const void* src_raw, const int64_t* seg_o
}
}
template <typename S, typename T>
void
SegmentSealedImpl::bulk_subscript_impl(const ColumnBase* column,
const int64_t* seg_offsets,
int64_t count,
void* dst_raw) {
auto field = reinterpret_cast<const VariableColumn<S>*>(column);
auto dst = reinterpret_cast<T*>(dst_raw);
for (int64_t i = 0; i < count; ++i) {
auto offset = seg_offsets[i];
if (offset != INVALID_SEG_OFFSET) {
dst[i] = T(field->raw_at(offset));
}
}
}
// for vector
void
SegmentSealedImpl::bulk_subscript_impl(
@ -512,10 +559,30 @@ SegmentSealedImpl::bulk_subscript(FieldId field_id, const int64_t* seg_offsets,
}
Assert(get_bit(field_data_ready_bitset_, field_id));
auto field_data = insert_record_.get_field_data_base(field_id);
AssertInfo(field_data->num_chunk() == 1, std::string("num chunk not equal to 1 for sealed segment, num_chunk: ") +
std::to_string(field_data->num_chunk()));
auto src_vec = field_data->get_chunk_data(0);
if (datatype_is_variable(field_meta.get_data_type())) {
switch (field_meta.get_data_type()) {
case DataType::VARCHAR:
case DataType::STRING: {
FixedVector<std::string> output(count);
bulk_subscript_impl<std::string>(variable_fields_.at(field_id).get(), seg_offsets, count,
output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
case DataType::JSON: {
FixedVector<std::string> output(count);
bulk_subscript_impl<Json, std::string>(variable_fields_.at(field_id).get(), seg_offsets, count,
output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
default:
PanicInfo("unsupported data type: " + datatype_name(field_meta.get_data_type()));
}
}
auto src_vec = fixed_fields_.at(field_id).data();
switch (field_meta.get_data_type()) {
case DataType::BOOL: {
FixedVector<bool> output(count);
@ -552,11 +619,6 @@ SegmentSealedImpl::bulk_subscript(FieldId field_id, const int64_t* seg_offsets,
bulk_subscript_impl<double>(src_vec, seg_offsets, count, output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
case DataType::VARCHAR: {
FixedVector<std::string> output(count);
bulk_subscript_impl<std::string>(src_vec, seg_offsets, count, output.data());
return CreateScalarDataArrayFrom(output.data(), count, field_meta);
}
case DataType::VECTOR_FLOAT:
case DataType::VECTOR_BINARY: {

8
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -28,6 +28,8 @@
#include "SegmentSealed.h"
#include "TimestampIndex.h"
#include "index/ScalarIndex.h"
#include "common/Column.h"
#include "storage/FieldData.h"
namespace milvus::segcore {
@ -123,6 +125,10 @@ class SegmentSealedImpl : public SegmentSealed {
static void
bulk_subscript_impl(const void* src_raw, const int64_t* seg_offsets, int64_t count, void* dst_raw);
template <typename S, typename T = S>
static void
bulk_subscript_impl(const ColumnBase* field, const int64_t* seg_offsets, int64_t count, void* dst_raw);
static void
bulk_subscript_impl(
int64_t element_sizeof, const void* src_raw, const int64_t* seg_offsets, int64_t count, void* dst_raw);
@ -201,6 +207,8 @@ class SegmentSealedImpl : public SegmentSealed {
SchemaPtr schema_;
int64_t id_;
std::unordered_map<FieldId, Column> fixed_fields_;
std::unordered_map<FieldId, std::unique_ptr<ColumnBase>> variable_fields_;
};
inline SegmentSealedPtr

81
internal/core/src/segcore/Utils.cpp
View File

@ -10,8 +10,12 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "segcore/Utils.h"
#include <string>
#include "common/FieldMeta.h"
#include "common/Utils.h"
#include "index/ScalarIndex.h"
#include "storage/FieldData.h"
#include "storage/RemoteChunkManagerFactory.h"
#include "common/Common.h"
#include "storage/Util.h"
@ -20,7 +24,7 @@ namespace milvus::segcore {
void
ParsePksFromFieldData(std::vector<PkType>& pks, const DataArray& data) {
switch (DataType(data.type())) {
switch (static_cast<DataType>(data.type())) {
case DataType::INT64: {
auto source_data = reinterpret_cast<const int64_t*>(data.scalars().long_data().data().data());
std::copy_n(source_data, pks.size(), pks.data());
@ -101,13 +105,13 @@ CreateScalarDataArray(int64_t count, const FieldMeta& field_meta) {
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
auto scalar_array = data_array->mutable_scalars();
switch (data_type) {
case DataType::BOOL: {
auto obj = scalar_array->mutable_bool_data();
obj->mutable_data()->Resize(count, 0);
obj->mutable_data()->Resize(count, false);
break;
}
case DataType::INT8: {
@ -143,9 +147,18 @@ CreateScalarDataArray(int64_t count, const FieldMeta& field_meta) {
case DataType::VARCHAR: {
auto obj = scalar_array->mutable_string_data();
obj->mutable_data()->Reserve(count);
for (auto i = 0; i < count; i++) *(obj->mutable_data()->Add()) = std::string();
for (auto i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = std::string();
}
break;
}
case DataType::JSON: {
auto obj = scalar_array->mutable_json_data();
obj->mutable_data()->Reserve(count);
for (int i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = std::string();
}
}
default: {
PanicInfo("unsupported datatype");
}
@ -159,7 +172,7 @@ CreateVectorDataArray(int64_t count, const FieldMeta& field_meta) {
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
auto vector_array = data_array->mutable_vectors();
auto dim = field_meta.get_dim();
@ -190,7 +203,7 @@ CreateScalarDataArrayFrom(const void* data_raw, int64_t count, const FieldMeta&
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
auto scalar_array = data_array->mutable_scalars();
switch (data_type) {
@ -239,7 +252,17 @@ CreateScalarDataArrayFrom(const void* data_raw, int64_t count, const FieldMeta&
case DataType::VARCHAR: {
auto data = reinterpret_cast<const std::string*>(data_raw);
auto obj = scalar_array->mutable_string_data();
for (auto i = 0; i < count; i++) *(obj->mutable_data()->Add()) = data[i];
for (auto i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = data[i];
}
break;
}
case DataType::JSON: {
auto data = reinterpret_cast<const std::string*>(data_raw);
auto obj = scalar_array->mutable_json_data();
for (auto i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = data[i];
}
break;
}
default: {
@ -255,7 +278,7 @@ CreateVectorDataArrayFrom(const void* data_raw, int64_t count, const FieldMeta&
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
auto vector_array = data_array->mutable_vectors();
auto dim = field_meta.get_dim();
@ -300,7 +323,7 @@ MergeDataArray(std::vector<std::pair<milvus::SearchResult*, int64_t>>& result_of
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
for (auto& result_pair : result_offsets) {
auto src_field_data = result_pair.first->output_fields_data_[field_meta.get_id()].get();
@ -311,13 +334,13 @@ MergeDataArray(std::vector<std::pair<milvus::SearchResult*, int64_t>>& result_of
auto dim = field_meta.get_dim();
vector_array->set_dim(dim);
if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) {
auto data = src_field_data->vectors().float_vector().data().data();
auto data = VEC_FIELD_DATA(src_field_data, float).data();
auto obj = vector_array->mutable_float_vector();
obj->mutable_data()->Add(data + src_offset * dim, data + (src_offset + 1) * dim);
} else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) {
AssertInfo(dim % 8 == 0, "Binary vector field dimension is not a multiple of 8");
auto num_bytes = dim / 8;
auto data = src_field_data->vectors().binary_vector().data();
auto data = VEC_FIELD_DATA(src_field_data, binary);
auto obj = vector_array->mutable_binary_vector();
obj->assign(data + src_offset * num_bytes, num_bytes);
} else {
@ -329,45 +352,51 @@ MergeDataArray(std::vector<std::pair<milvus::SearchResult*, int64_t>>& result_of
auto scalar_array = data_array->mutable_scalars();
switch (data_type) {
case DataType::BOOL: {
auto data = src_field_data->scalars().bool_data().data().data();
auto data = FIELD_DATA(src_field_data, bool).data();
auto obj = scalar_array->mutable_bool_data();
*(obj->mutable_data()->Add()) = data[src_offset];
continue;
break;
}
case DataType::INT8:
case DataType::INT16:
case DataType::INT32: {
auto data = src_field_data->scalars().int_data().data().data();
auto data = FIELD_DATA(src_field_data, int).data();
auto obj = scalar_array->mutable_int_data();
*(obj->mutable_data()->Add()) = data[src_offset];
continue;
break;
}
case DataType::INT64: {
auto data = src_field_data->scalars().long_data().data().data();
auto obj = scalar_array->mutable_long_data();
*(obj->mutable_data()->Add()) = data[src_offset];
continue;
break;
}
case DataType::FLOAT: {
auto data = src_field_data->scalars().float_data().data().data();
auto data = FIELD_DATA(src_field_data, float).data();
auto obj = scalar_array->mutable_float_data();
*(obj->mutable_data()->Add()) = data[src_offset];
continue;
break;
}
case DataType::DOUBLE: {
auto data = src_field_data->scalars().double_data().data().data();
auto data = FIELD_DATA(src_field_data, double).data();
auto obj = scalar_array->mutable_double_data();
*(obj->mutable_data()->Add()) = data[src_offset];
continue;
break;
}
case DataType::VARCHAR: {
auto data = src_field_data->scalars().string_data();
auto& data = FIELD_DATA(src_field_data, string);
auto obj = scalar_array->mutable_string_data();
*(obj->mutable_data()->Add()) = data.data(src_offset);
continue;
*(obj->mutable_data()->Add()) = data[src_offset];
break;
}
case DataType::JSON: {
auto& data = FIELD_DATA(src_field_data, json);
auto obj = scalar_array->mutable_json_data();
*(obj->mutable_data()->Add()) = data[src_offset];
break;
}
default: {
PanicInfo("unsupported datatype");
PanicInfo("unsupported data type " + datatype_name(data_type));
}
}
}
@ -384,7 +413,7 @@ ReverseDataFromIndex(const index::IndexBase* index,
auto data_type = field_meta.get_data_type();
auto data_array = std::make_unique<DataArray>();
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(milvus::proto::schema::DataType(field_meta.get_data_type()));
data_array->set_type(static_cast<milvus::proto::schema::DataType>(field_meta.get_data_type()));
auto scalar_array = data_array->mutable_scalars();
switch (data_type) {
@ -518,7 +547,7 @@ LoadFieldDatasFromRemote(std::vector<std::string>& remote_files) {
}
AssertInfo(field_datas.size() == remote_files.size(), "inconsistent file num and raw data num!");
return field_datas;
return std::vector<storage::FieldDataPtr>(std::move(field_datas));
}
} // namespace milvus::segcore

3
internal/core/src/segcore/Utils.h
View File

@ -13,7 +13,7 @@
#include <exception>
#include <memory>
#include <stdexcept>
#include <stdlib.h>
#include <cstdlib>
#include <string>
#include <utility>
#include <vector>
@ -23,6 +23,7 @@
#include "segcore/InsertRecord.h"
#include "index/Index.h"
#include "storage/ChunkManager.h"
#include "storage/FieldData.h"
namespace milvus::segcore {

11
internal/core/src/storage/FieldData.h
View File

@ -19,6 +19,7 @@
#include <string>
#include <memory>
#include "common/Json.h"
#include "storage/FieldDataInterface.h"
namespace milvus::storage {
@ -41,6 +42,15 @@ class FieldData<std::string> : public FieldDataStringImpl {
}
};
template <>
class FieldData<milvus::Json> : public FieldDataStringImpl {
public:
static_assert(IsScalar<std::string> || std::is_same_v<std::string, PkType>);
explicit FieldData(DataType data_type, int64_t buffered_num_rows = 0)
: FieldDataStringImpl(data_type, buffered_num_rows) {
}
};
template <>
class FieldData<FloatVector> : public FieldDataImpl<float, false> {
public:
@ -67,5 +77,4 @@ class FieldData<BinaryVector> : public FieldDataImpl<uint8_t, false> {
};
using FieldDataPtr = std::shared_ptr<FieldDataBase>;
} // namespace milvus::storage

3
internal/core/src/storage/FieldDataFactory.cpp
View File

@ -15,6 +15,7 @@
// limitations under the License.
#include "storage/FieldDataFactory.h"
#include "common/Json.h"
#include "storage/Exception.h"
namespace milvus::storage {
@ -39,6 +40,8 @@ FieldDataFactory::CreateFieldData(const DataType& type, int64_t dim, int64_t tot
case DataType::STRING:
case DataType::VARCHAR:
return std::make_shared<FieldData<std::string>>(type, total_num_rows);
case DataType::JSON:
return std::make_shared<FieldData<milvus::Json>>(type, total_num_rows);
case DataType::VECTOR_FLOAT:
return std::make_shared<FieldData<FloatVector>>(dim, type, total_num_rows);
case DataType::VECTOR_BINARY:

8
internal/core/src/storage/PayloadWriter.cpp
View File

@ -52,6 +52,14 @@ PayloadWriter::add_one_string_payload(const char* str, int str_size) {
rows_.fetch_add(1);
}
void
PayloadWriter::add_one_binary_payload(const uint8_t* data, int length) {
AssertInfo(output_ == nullptr, "payload writer has been finished");
AssertInfo(milvus::datatype_is_binary(column_type_), "mismatch data type");
AddOneBinaryToArrowBuilder(builder_, data, length);
rows_.fetch_add(1);
}
void
PayloadWriter::add_payload(const Payload& raw_data) {
AssertInfo(output_ == nullptr, "payload writer has been finished");

3
internal/core/src/storage/PayloadWriter.h
View File

@ -35,6 +35,9 @@ class PayloadWriter {
void
add_one_string_payload(const char* str, int str_size);
void
add_one_binary_payload(const uint8_t* data, int length);
void
finish();

44
internal/core/src/storage/Util.cpp
View File

@ -15,13 +15,16 @@
// limitations under the License.
#include "storage/Util.h"
#include "arrow/array/builder_binary.h"
#include "arrow/type_fwd.h"
#include "exceptions/EasyAssert.h"
#include "common/Consts.h"
#include "config/ConfigChunkManager.h"
#include "storage/FieldDataFactory.h"
#include "storage/MemFileManagerImpl.h"
#include "storage/FieldData.h"
#include "storage/parquet_c.h"
#include "storage/ThreadPool.h"
#include "log/Log.h"
#include "storage/MemFileManagerImpl.h"
#include "storage/FieldDataFactory.h"
#ifdef BUILD_DISK_ANN
#include "storage/DiskFileManagerImpl.h"
@ -129,6 +132,19 @@ AddOneStringToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder, const c
AssertInfo(ast.ok(), "append value to arrow builder failed");
}
void
AddOneBinaryToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder, const uint8_t* data, int length) {
AssertInfo(builder != nullptr, "empty arrow builder");
auto binary_builder = std::dynamic_pointer_cast<arrow::BinaryBuilder>(builder);
arrow::Status ast;
if (data == nullptr || length < 0) {
ast = binary_builder->AppendNull();
} else {
ast = binary_builder->Append(data, length);
}
AssertInfo(ast.ok(), "append value to arrow builder failed");
}
std::shared_ptr<arrow::ArrayBuilder>
CreateArrowBuilder(DataType data_type) {
switch (static_cast<DataType>(data_type)) {
@ -157,6 +173,10 @@ CreateArrowBuilder(DataType data_type) {
case DataType::STRING: {
return std::make_shared<arrow::StringBuilder>();
}
case DataType::ARRAY:
case DataType::JSON: {
return std::make_shared<arrow::BinaryBuilder>();
}
default: {
PanicInfo("unsupported numeric data type");
}
@ -208,6 +228,10 @@ CreateArrowSchema(DataType data_type) {
case DataType::STRING: {
return arrow::schema({arrow::field("val", arrow::utf8())});
}
case DataType::ARRAY:
case DataType::JSON: {
return arrow::schema({arrow::field("val", arrow::binary())});
}
default: {
PanicInfo("unsupported numeric data type");
}
@ -440,15 +464,23 @@ PutIndexData(RemoteChunkManager* remote_chunk_manager,
}
int64_t
GetTotalNumRowsForFieldDatas(const std::vector<FieldDataPtr>& field_datas) {
GetTotalNumRowsForFieldDatas(const std::vector<FieldDataPtr>& field_data) {
int64_t count = 0;
for (auto& field_data : field_datas) {
for (auto& field_data : field_data) {
count += field_data->get_num_rows();
}
return count;
}
size_t
CalcFieldDataSize(const std::vector<FieldDataPtr>& field_data) {
size_t size{};
for (auto& field_data : field_data) {
size += field_data->Size();
}
return size;
}
void
ReleaseArrowUnused() {
static std::mutex release_mutex;

53
internal/core/src/storage/Util.h
View File

@ -40,6 +40,8 @@ AddPayloadToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder, const Pay
void
AddOneStringToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder, const char* str, int str_size);
void
AddOneBinaryToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder, const uint8_t* data, int length);
std::shared_ptr<arrow::ArrayBuilder>
CreateArrowBuilder(DataType data_type);
@ -118,9 +120,60 @@ PutIndexData(RemoteChunkManager* remote_chunk_manager,
int64_t
GetTotalNumRowsForFieldDatas(const std::vector<FieldDataPtr>& field_datas);
size_t
CalcFieldDataSize(const std::vector<FieldDataPtr>& field_data);
void
ReleaseArrowUnused();
inline void
FillField(DataType data_type, size_t size, const std::vector<storage::FieldDataPtr>& field_data, void* dst) {
auto dest = static_cast<char*>(dst);
if (datatype_is_variable(data_type)) {
for (auto& data : field_data) {
auto n = data->get_num_rows();
for (auto i = 0; i < n; i++) {
auto src = static_cast<const std::string*>(data->RawValue(i));
std::copy_n(src->data(), src->size(), dest);
dest += src->size();
}
}
} else {
for (auto& data : field_data) {
std::copy_n(static_cast<const char*>(data->Data()), data->Size(), dest);
dest += data->Size();
}
}
}
// CreateMap creates a memory mapping,
// if mmap enabled, this writes field data to disk and create a map to the file,
// otherwise this just alloc memory
inline void*
CreateMap(int64_t segment_id, const FieldMeta& field_meta, const std::vector<storage::FieldDataPtr>& field_data) {
static int mmap_flags = MAP_PRIVATE;
#ifdef MAP_POPULATE
// macOS doesn't support MAP_POPULATE
mmap_flags |= MAP_POPULATE;
#endif
// simdjson requires a padding following the json data
size_t padding = field_meta.get_data_type() == DataType::JSON ? simdjson::SIMDJSON_PADDING : 0;
auto data_type = field_meta.get_data_type();
auto data_size = storage::CalcFieldDataSize(field_data);
if (data_size == 0) {
return nullptr;
}
// Use anon mapping so we are able to free these memory with munmap only
void* map = mmap(nullptr, data_size + padding, PROT_READ | PROT_WRITE, mmap_flags | MAP_ANON, -1, 0);
AssertInfo(map != MAP_FAILED, std::string("failed to create anon map, err: ") + strerror(errno));
FillField(data_type, data_size, field_data, map);
return map;
}
// size_t
// getCurrentRSS();
} // namespace milvus::storage

22
internal/core/src/storage/parquet_c.cpp
View File

@ -117,6 +117,28 @@ AddOneStringToPayload(CPayloadWriter payloadWriter, char* cstr, int str_size) {
}
}
extern "C" CStatus
AddOneArrayToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) {
try {
auto p = reinterpret_cast<PayloadWriter*>(payloadWriter);
p->add_one_binary_payload(data, length);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
}
}
extern "C" CStatus
AddOneJSONToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) {
try {
auto p = reinterpret_cast<PayloadWriter*>(payloadWriter);
p->add_one_binary_payload(data, length);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
}
}
extern "C" CStatus
AddBinaryVectorToPayload(CPayloadWriter payloadWriter, uint8_t* values, int dimension, int length) {
try {

4
internal/core/src/storage/parquet_c.h
View File

@ -53,6 +53,10 @@ AddDoubleToPayload(CPayloadWriter payloadWriter, double* values, int length);
CStatus
AddOneStringToPayload(CPayloadWriter payloadWriter, char* cstr, int str_size);
CStatus
AddOneArrayToPayload(CPayloadWriter payloadWriter, uint8_t* cdata, int length);
CStatus
AddOneJSONToPayload(CPayloadWriter payloadWriter, uint8_t* cdata, int length);
CStatus
AddBinaryVectorToPayload(CPayloadWriter payloadWriter, uint8_t* values, int dimension, int length);
CStatus
AddFloatVectorToPayload(CPayloadWriter payloadWriter, float* values, int dimension, int length);

136
internal/core/thirdparty/CMakeLists.txt vendored
View File

@ -1,89 +1,105 @@
#-------------------------------------------------------------------------------
# -------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
# Using default c and cxx compiler in our build tree
# Thirdpart cxx and c flags
add_compile_options( -O3 -fPIC -Wno-error -fopenmp )
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
# -------------------------------------------------------------------------------
# Using default c and cxx compiler in our build tree Thirdpart cxx and c flags
add_compile_options(-O3 -fPIC -Wno-error -fopenmp)
if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
else ()
set( EP_LOG_OPTIONS )
endif ()
set( MAKE_BUILD_ARGS "-j6" )
include( FetchContent )
set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
set( FETCHCONTENT_QUIET OFF )
if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
if(NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
set(EP_LOG_OPTIONS
LOG_CONFIGURE
1
LOG_BUILD
1
LOG_INSTALL
1
LOG_DOWNLOAD
1)
else()
set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
set(EP_LOG_OPTIONS)
endif()
message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
set(MAKE_BUILD_ARGS "-j6")
include(FetchContent)
set(FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download)
set(FETCHCONTENT_QUIET OFF)
if(CUSTOM_THIRDPARTY_DOWNLOAD_PATH)
set(THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH})
else()
set(THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download)
endif()
message(STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}")
# ----------------------------------------------------------------------
# Find pthreads
set( THREADS_PREFER_PTHREAD_FLAG ON )
find_package( Threads REQUIRED )
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
add_subdirectory( knowhere )
add_subdirectory(knowhere)
# ****************************** Thirdparty googletest ***************************************
if ( MILVUS_BUILD_TESTS)
add_subdirectory( gtest )
add_subdirectory( google_benchmark )
# ****************************** Thirdparty googletest
# ***************************************
if(MILVUS_BUILD_TESTS)
add_subdirectory(gtest)
add_subdirectory(google_benchmark)
endif()
if ( MILVUS_BUILD_TESTS AND LINUX )
add_subdirectory( profiler )
if(MILVUS_BUILD_TESTS AND LINUX)
add_subdirectory(profiler)
endif()
# ****************************** Thirdparty yaml
# ***************************************
add_subdirectory(yaml-cpp)
# ****************************** Thirdparty yaml ***************************************
add_subdirectory( yaml-cpp )
# ****************************** Thirdparty opentracing ***************************************
if ( MILVUS_WITH_OPENTRACING )
add_subdirectory( opentracing )
# ****************************** Thirdparty opentracing
# ***************************************
if(MILVUS_WITH_OPENTRACING)
add_subdirectory(opentracing)
endif()
add_subdirectory( protobuf )
add_subdirectory( boost_ext )
add_subdirectory( arrow )
add_subdirectory( rocksdb )
add_subdirectory(protobuf)
add_subdirectory(boost_ext)
add_subdirectory(arrow)
add_subdirectory(rocksdb)
add_subdirectory(simdjson)
# ************************ Thirdparty aws sdk and google cloud sdk **************************
if ( LINUX OR APPLE )
add_subdirectory( aws_sdk )
add_subdirectory( google_cloud_sdk )
# ************************ Thirdparty aws sdk and google cloud sdk
# **************************
if(LINUX OR APPLE)
add_subdirectory(aws_sdk)
add_subdirectory(google_cloud_sdk)
endif()
# ******************************* Thirdparty google cloud sdk ********************************
# ******************************* Thirdparty google cloud sdk
# ********************************
# ******************************* Thirdparty marisa ********************************
# ******************************* Thirdparty marisa
# ********************************
# TODO: support win.
if ( LINUX OR APPLE)
add_subdirectory( marisa )
if(LINUX OR APPLE)
add_subdirectory(marisa)
endif()
# ******************************* Thirdparty jemalloc ********************************
if ( LINUX )
add_subdirectory( jemalloc )
# ******************************* Thirdparty jemalloc
# ********************************
if(LINUX)
add_subdirectory(jemalloc)
endif()
# ******************************* Thirdparty rapidxml ********************************
#if ( LINUX )
# add_subdirectory( rapidxml )
#endif()
# ******************************* Thirdparty rapidxml
# ********************************
# if ( LINUX ) add_subdirectory( rapidxml ) endif()

19
internal/core/thirdparty/simdjson/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,19 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
FetchContent_Declare(
simdjson
GIT_REPOSITORY https://github.com/simdjson/simdjson.git
GIT_TAG v3.1.7
)
FetchContent_MakeAvailable(simdjson)

241
internal/core/unittest/test_expr.cpp
View File

@ -11,14 +11,22 @@
#include <boost/format.hpp>
#include <gtest/gtest.h>
#include <cstdint>
#include <memory>
#include <regex>
#include <vector>
#include "common/Json.h"
#include "common/Types.h"
#include "pb/plan.pb.h"
#include "query/Expr.h"
#include "query/ExprImpl.h"
#include "query/Plan.h"
#include "query/PlanNode.h"
#include "query/generated/ShowPlanNodeVisitor.h"
#include "query/generated/ExecExprVisitor.h"
#include "segcore/SegmentGrowingImpl.h"
#include "simdjson/padded_string.h"
#include "test_utils/DataGen.h"
#include "index/IndexFactory.h"
@ -326,6 +334,74 @@ TEST(Expr, TestRange) {
}
}
TEST(Expr, TestBinaryRangeJSON) {
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
bool lower_inclusive;
bool upper_inclusive;
int64_t lower;
int64_t upper;
std::vector<std::string> nested_path;
};
std::vector<Testcase> testcases{
{true, false, 10, 20, {"int"}},
{true, true, 20, 30, {"int"}},
{false, true, 30, 40, {"int"}},
{false, false, 40, 50, {"int"}},
};
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
schema->set_primary_field_id(i64_fid);
auto seg = CreateGrowingSegment(schema);
int N = 1000;
std::vector<std::string> json_col;
int num_iters = 100;
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_json_col = raw_data.get_col<std::string>(json_fid);
json_col.insert(json_col.end(), new_json_col.begin(), new_json_col.end());
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);
}
auto seg_promote = dynamic_cast<SegmentGrowingImpl*>(seg.get());
ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP);
for (auto testcase : testcases) {
auto check = [&](int64_t value) {
int64_t lower = testcase.lower, upper = testcase.upper;
if (!testcase.lower_inclusive) {
lower++;
}
if (!testcase.upper_inclusive) {
upper--;
}
return lower <= value && value <= upper;
};
RetrievePlanNode plan;
plan.predicate_ = std::make_unique<BinaryRangeExprImpl<int64_t>>(
ColumnInfo(json_fid, DataType::JSON, testcase.nested_path), proto::plan::GenericValue::ValCase::kInt64Val,
testcase.lower_inclusive, testcase.upper_inclusive, testcase.lower, testcase.upper);
auto final = visitor.call_child(*plan.predicate_);
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto ans = final[i];
auto val =
milvus::Json(simdjson::padded_string(json_col[i])).template at<int64_t>(testcase.nested_path).value();
auto ref = check(val);
ASSERT_EQ(ans, ref) << val << testcase.lower_inclusive << testcase.lower << testcase.upper_inclusive
<< testcase.upper;
}
}
}
TEST(Expr, TestTerm) {
using namespace milvus::query;
using namespace milvus::segcore;
@ -410,10 +486,9 @@ TEST(Expr, TestTerm) {
}
TEST(Expr, TestSimpleDsl) {
using namespace milvus::query;
using namespace milvus::segcore;
auto vec_dsl = Json::parse(R"({
auto vec_dsl = query::Json::parse(R"({
"vector": {
"fakevec": {
"metric_type": "L2",
@ -436,40 +511,40 @@ TEST(Expr, TestSimpleDsl) {
terms.push_back(i);
}
}
Json s;
query::Json s;
s["term"]["age"]["values"] = terms;
return s;
};
// std::cout << get_item(0).dump(-2);
// std::cout << vec_dsl.dump(-2);
std::vector<std::tuple<Json, std::function<bool(int)>>> testcases;
std::vector<std::tuple<query::Json, std::function<bool(int)>>> testcases;
{
Json dsl;
dsl["must"] = Json::array({vec_dsl, get_item(0), get_item(1), get_item(2, 0), get_item(3)});
query::Json dsl;
dsl["must"] = query::Json::array({vec_dsl, get_item(0), get_item(1), get_item(2, 0), get_item(3)});
testcases.emplace_back(dsl, [](int64_t x) { return (x & 0b1111) == 0b1011; });
}
{
Json dsl;
Json sub_dsl;
sub_dsl["must"] = Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = Json::array({sub_dsl, vec_dsl});
query::Json dsl;
query::Json sub_dsl;
sub_dsl["must"] = query::Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = query::Json::array({sub_dsl, vec_dsl});
testcases.emplace_back(dsl, [](int64_t x) { return (x & 0b1111) == 0b1011; });
}
{
Json dsl;
Json sub_dsl;
sub_dsl["should"] = Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = Json::array({sub_dsl, vec_dsl});
query::Json dsl;
query::Json sub_dsl;
sub_dsl["should"] = query::Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = query::Json::array({sub_dsl, vec_dsl});
testcases.emplace_back(dsl, [](int64_t x) { return !!((x & 0b1111) ^ 0b0100); });
}
{
Json dsl;
Json sub_dsl;
sub_dsl["must_not"] = Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = Json::array({sub_dsl, vec_dsl});
query::Json dsl;
query::Json sub_dsl;
sub_dsl["must_not"] = query::Json::array({get_item(0), get_item(1), get_item(2, 0), get_item(3)});
dsl["must"] = query::Json::array({sub_dsl, vec_dsl});
testcases.emplace_back(dsl, [](int64_t x) { return (x & 0b1111) != 0b1011; });
}
@ -490,12 +565,12 @@ TEST(Expr, TestSimpleDsl) {
}
auto seg_promote = dynamic_cast<SegmentGrowingImpl*>(seg.get());
ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP);
query::ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP);
for (auto [clause, ref_func] : testcases) {
Json dsl;
query::Json dsl;
dsl["bool"] = clause;
// std::cout << dsl.dump(2);
auto plan = CreatePlan(*schema, dsl.dump());
auto plan = query::CreatePlan(*schema, dsl.dump());
auto final = visitor.call_child(*plan->plan_node_->predicate_.value());
EXPECT_EQ(final.size(), N * num_iters);
@ -1015,6 +1090,130 @@ TEST(Expr, TestBinaryArithOpEvalRange) {
}
}
TEST(Expr, TestBinaryArithOpEvalRangeJSON) {
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
int64_t right_operand;
int64_t value;
OpType op;
std::vector<std::string> nested_path;
};
std::vector<Testcase> testcases{
{10, 20, OpType::Equal, {"int"}},
{20, 30, OpType::Equal, {"int"}},
{30, 40, OpType::NotEqual, {"int"}},
{40, 50, OpType::NotEqual, {"int"}},
};
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
schema->set_primary_field_id(i64_fid);
auto seg = CreateGrowingSegment(schema);
int N = 1000;
std::vector<std::string> json_col;
int num_iters = 100;
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_json_col = raw_data.get_col<std::string>(json_fid);
json_col.insert(json_col.end(), new_json_col.begin(), new_json_col.end());
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);
}
auto seg_promote = dynamic_cast<SegmentGrowingImpl*>(seg.get());
ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP);
for (auto testcase : testcases) {
auto check = [&](int64_t value) {
if (testcase.op == OpType::Equal) {
return value + testcase.right_operand == testcase.value;
}
return value + testcase.right_operand != testcase.value;
};
RetrievePlanNode plan;
plan.predicate_ = std::make_unique<BinaryArithOpEvalRangeExprImpl<int64_t>>(
ColumnInfo(json_fid, DataType::JSON, testcase.nested_path), proto::plan::GenericValue::ValCase::kInt64Val,
ArithOpType::Add, testcase.right_operand, testcase.op, testcase.value);
auto final = visitor.call_child(*plan.predicate_);
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto ans = final[i];
auto val =
milvus::Json(simdjson::padded_string(json_col[i])).template at<int64_t>(testcase.nested_path).value();
auto ref = check(val);
ASSERT_EQ(ans, ref) << testcase.value << " " << val;
}
}
}
TEST(Expr, TestBinaryArithOpEvalRangeJSONFloat) {
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
double right_operand;
double value;
OpType op;
std::vector<std::string> nested_path;
};
std::vector<Testcase> testcases{
{10, 20, OpType::Equal, {"double"}},
{20, 30, OpType::Equal, {"double"}},
{30, 40, OpType::NotEqual, {"double"}},
{40, 50, OpType::NotEqual, {"double"}},
};
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
schema->set_primary_field_id(i64_fid);
auto seg = CreateGrowingSegment(schema);
int N = 1000;
std::vector<std::string> json_col;
int num_iters = 100;
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_json_col = raw_data.get_col<std::string>(json_fid);
json_col.insert(json_col.end(), new_json_col.begin(), new_json_col.end());
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);
}
auto seg_promote = dynamic_cast<SegmentGrowingImpl*>(seg.get());
ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP);
for (auto testcase : testcases) {
auto check = [&](double value) {
if (testcase.op == OpType::Equal) {
return value + testcase.right_operand == testcase.value;
}
return value + testcase.right_operand != testcase.value;
};
RetrievePlanNode plan;
plan.predicate_ = std::make_unique<BinaryArithOpEvalRangeExprImpl<double>>(
ColumnInfo(json_fid, DataType::JSON, testcase.nested_path), proto::plan::GenericValue::ValCase::kFloatVal,
ArithOpType::Add, testcase.right_operand, testcase.op, testcase.value);
auto final = visitor.call_child(*plan.predicate_);
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto ans = final[i];
auto val =
milvus::Json(simdjson::padded_string(json_col[i])).template at<double>(testcase.nested_path).value();
auto ref = check(val);
ASSERT_EQ(ans, ref) << testcase.value << " " << val;
}
}
}
TEST(Expr, TestBinaryArithOpEvalRangeExceptions) {
using namespace milvus::query;
using namespace milvus::segcore;

8
internal/core/unittest/test_query.cpp
View File

@ -185,7 +185,7 @@ TEST(Query, ExecWithPredicateLoader) {
auto sr = segment->Search(plan.get(), ph_group.get(), time);
int topk = 5;
Json json = SearchResultToJson(*sr);
query::Json json = SearchResultToJson(*sr);
#ifdef __linux__
auto ref = json::parse(R"(
[
@ -263,7 +263,7 @@ TEST(Query, ExecWithPredicateSmallN) {
auto sr = segment->Search(plan.get(), ph_group.get(), time);
int topk = 5;
Json json = SearchResultToJson(*sr);
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
}
@ -317,7 +317,7 @@ TEST(Query, ExecWithPredicate) {
auto sr = segment->Search(plan.get(), ph_group.get(), time);
int topk = 5;
Json json = SearchResultToJson(*sr);
query::Json json = SearchResultToJson(*sr);
#ifdef __linux__
auto ref = json::parse(R"(
[
@ -806,7 +806,7 @@ TEST(Query, ExecWithPredicateBinary) {
auto sr = segment->Search(plan.get(), ph_group.get(), time);
int topk = 5;
Json json = SearchResultToJson(*sr);
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
// ASSERT_EQ(json.dump(2), ref.dump(2));
}

53
internal/core/unittest/test_retrieve.cpp
View File

@ -139,6 +139,59 @@ TEST(Retrieve, AutoID2) {
ASSERT_EQ(field1_data.data_size(), DIM * req_size);
}
TEST(Retrieve, String) {
auto schema = std::make_shared<Schema>();
auto DIM = 16;
auto fid_vec = schema->AddDebugField("vector_64", DataType::VECTOR_FLOAT, DIM, knowhere::metric::L2);
auto fid_str = schema->AddDebugField("str", DataType::VARCHAR);
schema->set_primary_field_id(fid_str);
int64_t N = 100;
int64_t req_size = 10;
auto choose = [=](int i) { return i * 3 % N; };
auto dataset = DataGen(schema, N);
auto segment = CreateSealedSegment(schema);
SealedLoadFieldData(dataset, *segment);
auto i64_col = dataset.get_col<std::string>(fid_str);
auto plan = std::make_unique<query::RetrievePlan>(*schema);
std::vector<std::string> values;
for (int i = 0; i < req_size; ++i) {
values.emplace_back(i64_col[choose(i)]);
}
auto term_expr = std::make_unique<query::TermExprImpl<std::string>>(fid_str, DataType::VARCHAR, values);
plan->plan_node_ = std::make_unique<query::RetrievePlanNode>();
plan->plan_node_->predicate_ = std::move(term_expr);
std::vector<FieldId> target_fields_id{
fid_str,
fid_vec,
};
plan->field_ids_ = target_fields_id;
auto retrieve_results = segment->Retrieve(plan.get(), 100);
Assert(retrieve_results->fields_data_size() == target_fields_id.size());
auto field0 = retrieve_results->fields_data(0);
Assert(field0.has_scalars());
auto field0_data = field0.scalars().string_data();
for (int i = 0; i < req_size; ++i) {
auto index = choose(i);
auto data = field0_data.data(i);
}
for (int i = 0; i < req_size; ++i) {
auto index = choose(i);
auto data = field0_data.data(i);
ASSERT_EQ(data, i64_col[index]);
}
auto field1 = retrieve_results->fields_data(1);
Assert(field1.has_vectors());
auto field1_data = field1.vectors().float_vector();
ASSERT_EQ(field1_data.data_size(), DIM * req_size);
}
TEST(Retrieve, NotExist) {
auto schema = std::make_shared<Schema>();
auto fid_64 = schema->AddDebugField("i64", DataType::INT64);

37
internal/core/unittest/test_sealed.cpp
View File

@ -354,6 +354,10 @@ TEST(Sealed, LoadFieldData) {
auto double_id = schema->AddDebugField("double", DataType::DOUBLE);
auto nothing_id = schema->AddDebugField("nothing", DataType::INT32);
auto str_id = schema->AddDebugField("str", DataType::VARCHAR);
schema->AddDebugField("int8", DataType::INT8);
schema->AddDebugField("int16", DataType::INT16);
schema->AddDebugField("float", DataType::FLOAT);
schema->AddDebugField("json", DataType::JSON);
schema->set_primary_field_id(counter_id);
auto dataset = DataGen(schema, N);
@ -417,7 +421,7 @@ TEST(Sealed, LoadFieldData) {
ASSERT_EQ(segment->num_chunk_index(str_id), 0);
auto chunk_span1 = segment->chunk_data<int64_t>(counter_id, 0);
auto chunk_span2 = segment->chunk_data<double>(double_id, 0);
auto chunk_span3 = segment->chunk_data<std::string>(str_id, 0);
auto chunk_span3 = segment->chunk_data<std::string_view>(str_id, 0);
auto ref1 = dataset.get_col<int64_t>(counter_id);
auto ref2 = dataset.get_col<double>(double_id);
auto ref3 = dataset.get_col(str_id)->scalars().string_data().data();
@ -433,36 +437,6 @@ TEST(Sealed, LoadFieldData) {
segment->DropIndex(fakevec_id);
ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group.get(), time));
// segment->LoadIndex(vec_info);
// auto sr2 = segment->Search(plan.get(), ph_group.get(), time);
// auto json2 = SearchResultToJson(*sr);
// ASSERT_EQ(json.dump(-2), json2.dump(-2));
// segment->DropFieldData(double_id);
// ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group.get(), time));
//#ifdef __linux__
// auto std_json = Json::parse(R"(
//[
// [
// ["982->0.000000", "25315->4.742000", "57893->4.758000", "48201->6.075000", "53853->6.223000"],
// ["41772->10.111000", "74859->11.790000", "79777->11.842000", "3785->11.983000", "35888->12.193000"],
// ["59251->2.543000", "65551->4.454000", "72204->5.332000", "96905->5.479000", "87833->5.765000"],
// ["59219->5.458000", "21995->6.078000", "97922->6.764000", "25710->7.158000", "14048->7.294000"],
// ["66353->5.696000", "30664->5.881000", "41087->5.917000", "10393->6.633000", "90215->7.202000"]
// ]
//])");
//#else // for mac
// auto std_json = Json::parse(R"(
//[
// [
// ["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "86706->5.991000"],
// ["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
// ["30119->2.464000", "82365->4.725000", "74834->5.009000", "79995->5.725000", "33359->5.816000"],
// ["99625->6.129000", "86582->6.900000", "85934->7.792000", "60450->8.087000", "19257->8.530000"],
// ["37759->3.581000", "31292->5.780000", "98124->6.216000", "63535->6.439000", "11707->6.553000"]
// ]
//])");
//#endif
// ASSERT_EQ(std_json.dump(-2), json.dump(-2));
}
TEST(Sealed, LoadScalarIndex) {
@ -629,7 +603,6 @@ TEST(Sealed, Delete) {
LoadDeletedRecordInfo info = {timestamps.data(), ids.get(), row_count};
segment->LoadDeletedRecord(info);
std::vector<uint8_t> tmp_block{0, 0};
BitsetType bitset(N, false);
segment->mask_with_delete(bitset, 10, 11);
ASSERT_EQ(bitset.count(), pks.size());

16
internal/core/unittest/test_string_expr.cpp
View File

@ -262,8 +262,8 @@ TEST(StringExpr, Term) {
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_str_col = raw_data.get_col(str_meta.get_id());
auto begin = new_str_col->scalars().string_data().data().begin();
auto end = new_str_col->scalars().string_data().data().end();
auto begin = FIELD_DATA(new_str_col, string).begin();
auto end = FIELD_DATA(new_str_col, string).end();
str_col.insert(str_col.end(), begin, end);
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);
@ -336,8 +336,8 @@ TEST(StringExpr, Compare) {
auto reserve_col = [&, raw_data](const FieldMeta& field_meta, std::vector<std::string>& str_col) {
auto new_str_col = raw_data.get_col(field_meta.get_id());
auto begin = new_str_col->scalars().string_data().data().begin();
auto end = new_str_col->scalars().string_data().data().end();
auto begin = FIELD_DATA(new_str_col, string).begin();
auto end = FIELD_DATA(new_str_col, string).end();
str_col.insert(str_col.end(), begin, end);
};
@ -408,8 +408,8 @@ TEST(StringExpr, UnaryRange) {
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_str_col = raw_data.get_col(str_meta.get_id());
auto begin = new_str_col->scalars().string_data().data().begin();
auto end = new_str_col->scalars().string_data().data().end();
auto begin = FIELD_DATA(new_str_col, string).begin();
auto end = FIELD_DATA(new_str_col, string).end();
str_col.insert(str_col.end(), begin, end);
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);
@ -473,8 +473,8 @@ TEST(StringExpr, BinaryRange) {
for (int iter = 0; iter < num_iters; ++iter) {
auto raw_data = DataGen(schema, N, iter);
auto new_str_col = raw_data.get_col(str_meta.get_id());
auto begin = new_str_col->scalars().string_data().data().begin();
auto end = new_str_col->scalars().string_data().data().end();
auto begin = FIELD_DATA(new_str_col, string).begin();
auto end = FIELD_DATA(new_str_col, string).end();
str_col.insert(str_col.end(), begin, end);
seg->PreInsert(N);
seg->Insert(iter * N, N, raw_data.row_ids_.data(), raw_data.timestamps_.data(), raw_data.raw_);

67
internal/core/unittest/test_utils/DataGen.h
View File

@ -15,6 +15,7 @@
#include <cstring>
#include <memory>
#include <random>
#include <string>
#include <google/protobuf/text_format.h>
#include "Constants.h"
@ -143,6 +144,12 @@ struct GeneratedData {
break;
}
case DataType::JSON: {
auto ret_data = reinterpret_cast<std::string*>(ret.data());
auto src_data = target_field_data.scalars().json_data().data();
std::copy(src_data.begin(), src_data.end(), ret_data);
break;
}
default: {
PanicInfo("unsupported");
}
@ -280,6 +287,16 @@ DataGen(SchemaPtr schema, int64_t N, uint64_t seed = 42, uint64_t ts_offset = 0,
insert_cols(data, N, field_meta);
break;
}
case DataType::JSON: {
vector<std::string> data(N);
for (int i = 0; i < N / repeat_count; i++) {
auto str = R"({"int":)" + std::to_string(er()) + R"(,"double":)" +
std::to_string(static_cast<double>(er())) + "}";
data[i] = str;
}
insert_cols(data, N, field_meta);
break;
}
default: {
throw std::runtime_error("unimplemented");
}
@ -437,11 +454,17 @@ CreateFieldDataFromDataArray(ssize_t raw_count, const DataArray* data, const Fie
raw_data = data->vectors().float_vector().data().data();
dim = field_meta.get_dim();
data_type = DataType::VECTOR_FLOAT;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
} else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) {
raw_data = data->vectors().binary_vector().data();
dim = field_meta.get_dim();
AssertInfo(dim % 8 == 0, "wrong dim value for binary vector");
data_type = DataType::VECTOR_BINARY;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
} else {
PanicInfo("unsupported");
}
@ -450,7 +473,9 @@ CreateFieldDataFromDataArray(ssize_t raw_count, const DataArray* data, const Fie
case DataType::BOOL: {
raw_data = data->scalars().bool_data().data().data();
data_type = DataType::BOOL;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::INT8: {
auto src_data = data->scalars().int_data().data();
@ -458,7 +483,9 @@ CreateFieldDataFromDataArray(ssize_t raw_count, const DataArray* data, const Fie
std::copy_n(src_data.data(), src_data.size(), data_raw.data());
raw_data = data_raw.data();
data_type = DataType::INT8;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::INT16: {
auto src_data = data->scalars().int_data().data();
@ -466,27 +493,37 @@ CreateFieldDataFromDataArray(ssize_t raw_count, const DataArray* data, const Fie
std::copy_n(src_data.data(), src_data.size(), data_raw.data());
raw_data = data_raw.data();
data_type = DataType::INT16;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::INT32: {
raw_data = data->scalars().int_data().data().data();
data_type = DataType::INT32;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::INT64: {
raw_data = data->scalars().long_data().data().data();
data_type = DataType::INT64;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::FLOAT: {
raw_data = data->scalars().float_data().data().data();
data_type = DataType::FLOAT;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::DOUBLE: {
raw_data = data->scalars().double_data().data().data();
data_type = DataType::DOUBLE;
break;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::VARCHAR: {
auto begin = data->scalars().string_data().data().begin();
@ -498,15 +535,23 @@ CreateFieldDataFromDataArray(ssize_t raw_count, const DataArray* data, const Fie
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
case DataType::JSON: {
auto begin = data->scalars().json_data().data().begin();
auto end = data->scalars().json_data().data().end();
std::vector<std::string> data_raw(begin, end);
raw_data = data_raw.data();
data_type = DataType::JSON;
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
}
default: {
PanicInfo("unsupported");
}
}
}
auto field_data = storage::FieldDataFactory::GetInstance().CreateFieldData(data_type, dim);
field_data->FillFieldData(raw_data, raw_count);
return field_data;
return {};
}
inline void
@ -529,7 +574,7 @@ SealedLoadFieldData(const GeneratedData& dataset, SegmentSealed& seg, const std:
}
}
auto fields = dataset.schema_->get_fields();
for (auto field_data : dataset.raw_->fields_data()) {
for (auto& field_data : dataset.raw_->fields_data()) {
int64_t field_id = field_data.field_id();
if (exclude_fields.find(field_id) != exclude_fields.end()) {
continue;

5
internal/datanode/flow_graph_insert_buffer_node.go
View File

@ -23,6 +23,7 @@ import (
"reflect"
"sync"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/opentracing/opentracing-go"
"go.uber.org/atomic"
@ -182,7 +183,7 @@ func (ibNode *insertBufferNode) Operate(in []Msg) []Msg {
seg2Upload, err := ibNode.addSegmentAndUpdateRowNum(fgMsg.insertMessages, startPositions[0], endPositions[0])
if err != nil {
// Occurs only if the collectionID is mismatch, should not happen
err = fmt.Errorf("update segment states in channel meta wrong, err = %s", err)
err = errors.Wrap(err, "update segment states in channel meta wrong")
log.Error(err.Error())
panic(err)
}
@ -192,7 +193,7 @@ func (ibNode *insertBufferNode) Operate(in []Msg) []Msg {
err := ibNode.bufferInsertMsg(msg, startPositions[0], endPositions[0])
if err != nil {
// error occurs when missing schema info or data is misaligned, should not happen
err = fmt.Errorf("insertBufferNode msg to buffer failed, err = %s", err)
err = errors.Wrap(err, "insertBufferNode msg to buffer failed")
log.Error(err.Error())
panic(err)
}

2
internal/distributed/proxy/service.go
View File

@ -935,7 +935,7 @@ func (s *Server) CreateDatabase(ctx context.Context, request *milvuspb.CreateDat
panic("implement me")
}
func (s *Server) DropDatabase(ctx context.Context, request *milvuspb.CreateDatabaseRequest) (*commonpb.Status, error) {
func (s *Server) DropDatabase(ctx context.Context, request *milvuspb.DropDatabaseRequest) (*commonpb.Status, error) {
//TODO implement me
panic("implement me")
}

1
internal/proto/plan.proto
View File

@ -50,6 +50,7 @@ message ColumnInfo {
schema.DataType data_type = 2;
bool is_primary_key = 3;
bool is_autoID = 4;
repeated string nested_path = 5;
}
message ColumnExpr {

188
internal/proto/planpb/plan.pb.go
View File

@ -347,6 +347,7 @@ type ColumnInfo struct {
DataType schemapb.DataType `protobuf:"varint,2,opt,name=data_type,json=dataType,proto3,enum=milvus.proto.schema.DataType" json:"data_type,omitempty"`
IsPrimaryKey bool `protobuf:"varint,3,opt,name=is_primary_key,json=isPrimaryKey,proto3" json:"is_primary_key,omitempty"`
IsAutoID bool `protobuf:"varint,4,opt,name=is_autoID,json=isAutoID,proto3" json:"is_autoID,omitempty"`
NestedPath []string `protobuf:"bytes,5,rep,name=nested_path,json=nestedPath,proto3" json:"nested_path,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -405,6 +406,13 @@ func (m *ColumnInfo) GetIsAutoID() bool {
return false
}
func (m *ColumnInfo) GetNestedPath() []string {
if m != nil {
return m.NestedPath
}
return nil
}
type ColumnExpr struct {
Info *ColumnInfo `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -1380,93 +1388,95 @@ func init() {
func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) }
var fileDescriptor_2d655ab2f7683c23 = []byte{
// 1407 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x57, 0xcd, 0x72, 0xdc, 0x44,
0x10, 0x5e, 0xad, 0x76, 0xbd, 0x52, 0xef, 0x7a, 0xad, 0xe8, 0x42, 0x7e, 0x48, 0x6c, 0x44, 0x8a,
0x98, 0x50, 0xb1, 0x2b, 0x24, 0x24, 0x95, 0x50, 0x01, 0xff, 0x05, 0x7b, 0x8b, 0xc4, 0x36, 0x8a,
0xe3, 0x03, 0x17, 0xd5, 0xac, 0x34, 0xf6, 0x4e, 0x45, 0xab, 0x51, 0x46, 0xa3, 0x4d, 0xf6, 0xcc,
0x8d, 0x1b, 0x0f, 0xc0, 0x15, 0xee, 0xdc, 0xe0, 0xc2, 0x0b, 0x70, 0xe0, 0xc8, 0x9d, 0xb7, 0xe0,
0x44, 0x4d, 0x8f, 0xf6, 0x2f, 0xb5, 0x1b, 0xaf, 0x8b, 0x54, 0x71, 0x1b, 0xf5, 0x74, 0x7f, 0xd3,
0xfd, 0x4d, 0x77, 0x4f, 0x0b, 0x20, 0x8d, 0x49, 0xb2, 0x96, 0x0a, 0x2e, 0xb9, 0x7b, 0xa1, 0xcb,
0xe2, 0x5e, 0x9e, 0xe9, 0xaf, 0x35, 0xb5, 0x71, 0xb9, 0x91, 0x85, 0x1d, 0xda, 0x25, 0x5a, 0xe4,
0xfd, 0x60, 0x40, 0x63, 0x97, 0x26, 0x54, 0xb0, 0xf0, 0x98, 0xc4, 0x39, 0x75, 0xaf, 0x80, 0xd5,
0xe6, 0x3c, 0x0e, 0x7a, 0x24, 0xbe, 0x68, 0xac, 0x18, 0xab, 0xd6, 0x5e, 0xc9, 0xaf, 0x29, 0xc9,
0x31, 0x89, 0xdd, 0xab, 0x60, 0xb3, 0x44, 0xde, 0xbb, 0x8b, 0xbb, 0xe5, 0x15, 0x63, 0xd5, 0xdc,
0x2b, 0xf9, 0x16, 0x8a, 0x8a, 0xed, 0x93, 0x98, 0x13, 0x89, 0xdb, 0xe6, 0x8a, 0xb1, 0x6a, 0xa8,
0x6d, 0x14, 0xa9, 0xed, 0x65, 0x80, 0x4c, 0x0a, 0x96, 0x9c, 0xe2, 0x7e, 0x65, 0xc5, 0x58, 0xb5,
0xf7, 0x4a, 0xbe, 0xad, 0x65, 0xc7, 0x24, 0xde, 0xaa, 0x82, 0xd9, 0x23, 0xb1, 0xf7, 0xbd, 0x01,
0xf6, 0x37, 0x39, 0x15, 0xfd, 0x56, 0x72, 0xc2, 0x5d, 0x17, 0x2a, 0x92, 0xa7, 0x2f, 0xd0, 0x19,
0xd3, 0xc7, 0xb5, 0xbb, 0x0c, 0xf5, 0x2e, 0x95, 0x82, 0x85, 0x81, 0xec, 0xa7, 0x14, 0x8f, 0xb2,
0x7d, 0xd0, 0xa2, 0xa3, 0x7e, 0x4a, 0xdd, 0x0f, 0x61, 0x31, 0xa3, 0x44, 0x84, 0x9d, 0x20, 0x25,
0x82, 0x74, 0x33, 0x7d, 0x9a, 0xdf, 0xd0, 0xc2, 0x43, 0x94, 0x29, 0x25, 0xc1, 0xf3, 0x24, 0x0a,
0x22, 0x1a, 0xb2, 0x2e, 0x89, 0x2f, 0x56, 0xf1, 0x88, 0x06, 0x0a, 0x77, 0xb4, 0xcc, 0xfb, 0xc9,
0x00, 0xd8, 0xe6, 0x71, 0xde, 0x4d, 0xd0, 0x9b, 0x4b, 0x60, 0x9d, 0x30, 0x1a, 0x47, 0x01, 0x8b,
0x0a, 0x8f, 0x6a, 0xf8, 0xdd, 0x8a, 0xdc, 0x87, 0x60, 0x47, 0x44, 0x12, 0xed, 0x92, 0x22, 0xa7,
0xf9, 0xe9, 0xd5, 0xb5, 0x09, 0xfe, 0x0b, 0xe6, 0x77, 0x88, 0x24, 0xca, 0x4b, 0xdf, 0x8a, 0x8a,
0x95, 0x7b, 0x1d, 0x9a, 0x2c, 0x0b, 0x52, 0xc1, 0xba, 0x44, 0xf4, 0x83, 0x17, 0xb4, 0x8f, 0x31,
0x59, 0x7e, 0x83, 0x65, 0x87, 0x5a, 0xf8, 0x35, 0xed, 0xbb, 0x57, 0xc0, 0x66, 0x59, 0x40, 0x72,
0xc9, 0x5b, 0x3b, 0x18, 0x91, 0xe5, 0x5b, 0x2c, 0xdb, 0xc4, 0x6f, 0xef, 0xcb, 0x81, 0x9f, 0x8f,
0x5f, 0xa7, 0xc2, 0xbd, 0x0d, 0x15, 0x96, 0x9c, 0x70, 0xf4, 0xb1, 0xfe, 0xa6, 0x1f, 0x98, 0x20,
0xa3, 0xa0, 0x7c, 0x54, 0xf5, 0xb6, 0xc0, 0xc6, 0x14, 0x40, 0xfb, 0xcf, 0xa0, 0xda, 0x53, 0x1f,
0x05, 0xc0, 0xf2, 0x14, 0x80, 0xf1, 0xb4, 0xf1, 0xb5, 0xb6, 0xf7, 0x8b, 0x01, 0xcd, 0xe7, 0x09,
0x11, 0x7d, 0x9f, 0x24, 0xa7, 0x1a, 0xe9, 0x0b, 0xa8, 0x87, 0x78, 0x54, 0x30, 0xbf, 0x43, 0x10,
0x8e, 0x18, 0xff, 0x18, 0xca, 0x3c, 0x2d, 0xf8, 0xbc, 0x34, 0xc5, 0xec, 0x20, 0x45, 0x2e, 0xcb,
0x3c, 0x1d, 0x39, 0x6d, 0x9e, 0xcb, 0xe9, 0x9f, 0xcb, 0xb0, 0xb4, 0xc5, 0xde, 0xad, 0xd7, 0x37,
0x60, 0x29, 0xe6, 0xaf, 0xa8, 0x08, 0x58, 0x12, 0xc6, 0x79, 0xc6, 0x7a, 0x3a, 0x25, 0x2c, 0xbf,
0x89, 0xe2, 0xd6, 0x40, 0xaa, 0x14, 0xf3, 0x34, 0x9d, 0x50, 0xd4, 0x57, 0xdf, 0x44, 0xf1, 0x48,
0x71, 0x03, 0xea, 0x1a, 0x51, 0x87, 0x58, 0x99, 0x2f, 0x44, 0x40, 0x1b, 0x5d, 0xda, 0x1b, 0x50,
0xd7, 0x47, 0x69, 0x84, 0xea, 0x9c, 0x08, 0x68, 0x83, 0x6b, 0xef, 0x0f, 0x03, 0xea, 0xdb, 0xbc,
0x9b, 0x12, 0xa1, 0x59, 0xda, 0x05, 0x27, 0xa6, 0x27, 0x32, 0x38, 0x37, 0x55, 0x4d, 0x65, 0x36,
0x56, 0x56, 0x2d, 0xb8, 0x20, 0xd8, 0x69, 0x67, 0x12, 0xa9, 0x3c, 0x0f, 0xd2, 0x12, 0xda, 0x6d,
0xbf, 0x99, 0x2f, 0xe6, 0x1c, 0xf9, 0xe2, 0x7d, 0x67, 0x80, 0x75, 0x44, 0x45, 0xf7, 0x9d, 0xdc,
0xf8, 0x7d, 0x58, 0x40, 0x5e, 0xb3, 0x8b, 0xe5, 0x15, 0x73, 0x1e, 0x62, 0x0b, 0x75, 0xd5, 0x82,
0x6d, 0xac, 0x19, 0x74, 0xe3, 0x2e, 0xba, 0x6f, 0xa0, 0xfb, 0xd7, 0xa7, 0x40, 0x0c, 0x35, 0xf5,
0xea, 0x20, 0xc5, 0xcc, 0xbf, 0x05, 0xd5, 0xb0, 0xc3, 0xe2, 0xa8, 0xe0, 0xec, 0xbd, 0x29, 0x86,
0xca, 0xc6, 0xd7, 0x5a, 0xde, 0x32, 0xd4, 0x0a, 0x6b, 0xb7, 0x0e, 0xb5, 0x56, 0xd2, 0x23, 0x31,
0x8b, 0x9c, 0x92, 0x5b, 0x03, 0x73, 0x9f, 0x4b, 0xc7, 0xf0, 0xfe, 0x32, 0x00, 0x74, 0x49, 0xa0,
0x53, 0xf7, 0xc6, 0x9c, 0xfa, 0x68, 0x0a, 0xf6, 0x48, 0xb5, 0x58, 0x16, 0x6e, 0x7d, 0x02, 0x15,
0x75, 0xd1, 0x67, 0x79, 0x85, 0x4a, 0x2a, 0x06, 0xbc, 0xcb, 0xa2, 0x7a, 0x67, 0xc7, 0x80, 0x5a,
0xde, 0x3d, 0xb0, 0x06, 0x67, 0x4d, 0x06, 0xd1, 0x04, 0x78, 0xc2, 0x4f, 0x59, 0x48, 0xe2, 0xcd,
0x24, 0x72, 0x0c, 0x77, 0x11, 0xec, 0xe2, 0xfb, 0x40, 0x38, 0x65, 0xef, 0x4f, 0x03, 0x16, 0xb5,
0xe1, 0xa6, 0x60, 0xb2, 0x73, 0x90, 0xfe, 0xe7, 0x9b, 0x7f, 0x00, 0x16, 0x51, 0x50, 0xc1, 0xb0,
0x4f, 0x5d, 0x9b, 0x62, 0x5c, 0x9c, 0x86, 0xc9, 0x57, 0x23, 0xc5, 0xd1, 0x3b, 0xb0, 0xa8, 0xf3,
0x9e, 0xa7, 0x54, 0x90, 0x24, 0x9a, 0xb7, 0x73, 0x35, 0xd0, 0xea, 0x40, 0x1b, 0x79, 0x3f, 0x1a,
0x83, 0x06, 0x86, 0x87, 0xe0, 0x95, 0x0d, 0xa8, 0x37, 0xce, 0x45, 0x7d, 0x79, 0x1e, 0xea, 0xdd,
0xb5, 0xb1, 0x12, 0x3b, 0x2b, 0x54, 0x55, 0x67, 0xbf, 0x97, 0xe1, 0xf2, 0x04, 0xe5, 0x8f, 0x7b,
0x24, 0x7e, 0x77, 0xbd, 0xf6, 0xff, 0xe6, 0xbf, 0x68, 0x39, 0x95, 0x73, 0x3d, 0x51, 0xd5, 0x73,
0x3d, 0x51, 0xff, 0x54, 0xa1, 0x82, 0x5c, 0x3d, 0x04, 0x5b, 0x52, 0xd1, 0x0d, 0xe8, 0xeb, 0x54,
0x14, 0x4c, 0x5d, 0x99, 0x82, 0x31, 0xe8, 0x6a, 0x6a, 0xfe, 0x92, 0x83, 0x0e, 0xf7, 0x08, 0x20,
0x57, 0x97, 0xa0, 0x8d, 0xf5, 0x55, 0xbf, 0xff, 0xb6, 0x16, 0xa3, 0xa6, 0xb3, 0x7c, 0xd8, 0x04,
0x36, 0xa0, 0xde, 0x66, 0x23, 0x7b, 0x73, 0xe6, 0x35, 0x8d, 0xba, 0xc1, 0x5e, 0xc9, 0x87, 0xf6,
0xa8, 0x8d, 0x6c, 0x43, 0x23, 0xd4, 0xaf, 0x87, 0x86, 0xd0, 0x6f, 0xd8, 0xb5, 0xa9, 0x37, 0x3d,
0x7c, 0x64, 0xf6, 0x4a, 0x7e, 0x3d, 0x1c, 0x7b, 0x73, 0x9e, 0x82, 0xa3, 0xa3, 0x10, 0x2a, 0x81,
0x34, 0x90, 0x26, 0xf3, 0x83, 0x59, 0xb1, 0x0c, 0x53, 0x6d, 0xaf, 0xe4, 0x37, 0xf3, 0xc9, 0x87,
0xfe, 0x10, 0x2e, 0x14, 0x51, 0x8d, 0xe1, 0x2d, 0x20, 0x9e, 0x37, 0x33, 0xb6, 0x71, 0xc0, 0xa5,
0xf6, 0x1b, 0xa3, 0x83, 0x84, 0xe5, 0x02, 0x71, 0x90, 0x95, 0x01, 0xed, 0x91, 0x78, 0x1c, 0xbf,
0x86, 0xf8, 0xb7, 0x66, 0xe2, 0x4f, 0x2b, 0x93, 0xbd, 0x92, 0x7f, 0xb9, 0x3d, 0xbb, 0x88, 0x46,
0x71, 0xe8, 0x53, 0xf1, 0x1c, 0xeb, 0x8c, 0x38, 0x86, 0xed, 0x62, 0x14, 0xc7, 0xa8, 0x83, 0x3c,
0x02, 0xc0, 0xe4, 0xd3, 0x50, 0xf6, 0xcc, 0x74, 0x19, 0x0e, 0x8d, 0x2a, 0x5d, 0x7a, 0xc3, 0x09,
0x72, 0x63, 0x58, 0xd5, 0x68, 0x0f, 0x67, 0x54, 0xf5, 0x20, 0x5d, 0xc2, 0xe1, 0xd7, 0xd6, 0x02,
0x54, 0x94, 0xa9, 0xf7, 0xb7, 0x01, 0x70, 0x4c, 0x43, 0xc9, 0xc5, 0xe6, 0xfe, 0xfe, 0xb3, 0x62,
0x0a, 0xd6, 0xde, 0xea, 0x5f, 0x14, 0x35, 0x05, 0xeb, 0x80, 0x26, 0xe6, 0xf3, 0xf2, 0xe4, 0x7c,
0x7e, 0x1f, 0x20, 0x15, 0x34, 0x62, 0x21, 0x91, 0x34, 0x3b, 0xeb, 0x91, 0x19, 0x53, 0x75, 0x3f,
0x07, 0x78, 0xa9, 0x7e, 0x47, 0x74, 0x7b, 0xaa, 0xcc, 0x24, 0x62, 0xf8, 0xcf, 0xe2, 0xdb, 0x2f,
0x87, 0xbf, 0x2f, 0x37, 0x60, 0x29, 0x8d, 0x49, 0x48, 0x3b, 0x3c, 0x8e, 0xa8, 0x08, 0x24, 0x39,
0xc5, 0x6c, 0xb5, 0xfd, 0xe6, 0x98, 0xf8, 0x88, 0x9c, 0x7a, 0xbf, 0x1a, 0x60, 0x1d, 0xc6, 0x24,
0xd9, 0xe7, 0x11, 0x8e, 0x6a, 0x3d, 0x8c, 0x38, 0x20, 0x49, 0x92, 0xbd, 0xa5, 0x25, 0x8e, 0x78,
0x51, 0xe4, 0x69, 0x9b, 0xcd, 0x24, 0xc9, 0xdc, 0x07, 0x13, 0xd1, 0xbe, 0xbd, 0xaf, 0x2b, 0xd3,
0xb1, 0x78, 0x57, 0xc1, 0xe1, 0xb9, 0x4c, 0x73, 0x19, 0x0c, 0xa8, 0x54, 0x74, 0x99, 0xab, 0xa6,
0xdf, 0xd4, 0xf2, 0xaf, 0x34, 0xa3, 0x99, 0xba, 0xa1, 0x84, 0x47, 0xf4, 0xe6, 0x6f, 0x06, 0x2c,
0xe8, 0x26, 0x37, 0xf9, 0x14, 0x2f, 0x41, 0x7d, 0x57, 0x50, 0x22, 0xa9, 0x38, 0xea, 0x90, 0xc4,
0x31, 0x5c, 0x07, 0x1a, 0x85, 0xe0, 0xf1, 0xcb, 0x9c, 0xc4, 0x4e, 0xd9, 0x6d, 0x80, 0xf5, 0x84,
0x66, 0x19, 0xee, 0x9b, 0xf8, 0x56, 0xd3, 0x2c, 0xd3, 0x9b, 0x15, 0xd7, 0x86, 0xaa, 0x5e, 0x56,
0x95, 0xde, 0x3e, 0x97, 0xfa, 0x6b, 0x41, 0x01, 0x1f, 0x0a, 0x7a, 0xc2, 0x5e, 0x3f, 0x25, 0x32,
0xec, 0x38, 0x35, 0x05, 0x7c, 0xc8, 0x33, 0x39, 0x94, 0x58, 0xca, 0x56, 0x2f, 0x6d, 0xb5, 0xc4,
0x42, 0x71, 0xc0, 0x5d, 0x80, 0x72, 0x2b, 0x71, 0xea, 0x4a, 0xb4, 0xcf, 0x65, 0x2b, 0x71, 0x1a,
0x37, 0x77, 0xa1, 0x3e, 0xf6, 0x36, 0xa8, 0x00, 0x9e, 0x27, 0x2f, 0x12, 0xfe, 0x2a, 0xd1, 0x03,
0xd1, 0x66, 0xa4, 0x86, 0x88, 0x1a, 0x98, 0xcf, 0xf2, 0xb6, 0x53, 0x56, 0x8b, 0xa7, 0x79, 0xec,
0x98, 0x6a, 0xb1, 0xc3, 0x7a, 0x4e, 0x05, 0x25, 0x3c, 0x72, 0xaa, 0x5b, 0x77, 0xbe, 0xbd, 0x7d,
0xca, 0x64, 0x27, 0x6f, 0xaf, 0x85, 0xbc, 0xbb, 0xae, 0xa9, 0xbe, 0xc5, 0x78, 0xb1, 0x5a, 0x67,
0x89, 0xa4, 0x22, 0x21, 0xf1, 0x3a, 0xb2, 0xbf, 0xae, 0xd8, 0x4f, 0xdb, 0xed, 0x05, 0xfc, 0xba,
0xf3, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x32, 0x23, 0x5b, 0x87, 0xb6, 0x0f, 0x00, 0x00,
// 1429 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x57, 0x3f, 0x73, 0xdb, 0xc6,
0x12, 0x27, 0x08, 0x52, 0x04, 0x96, 0x14, 0x05, 0xa3, 0x79, 0xb2, 0xfd, 0x6c, 0xe9, 0xe1, 0x79,
0x62, 0xc5, 0x19, 0x4b, 0xe3, 0xd8, 0xb1, 0xc7, 0xce, 0x38, 0xd1, 0x3f, 0x47, 0xe2, 0xc4, 0x96,
0x14, 0x58, 0x56, 0x91, 0x06, 0x73, 0x04, 0x4e, 0x22, 0xc6, 0xe0, 0x1d, 0x7c, 0x38, 0xd0, 0x66,
0x9d, 0x2e, 0x5d, 0x3e, 0x40, 0xea, 0xf4, 0xe9, 0x92, 0x26, 0x65, 0x9a, 0x14, 0x29, 0xd3, 0xe7,
0x5b, 0xa4, 0xca, 0xdc, 0x1e, 0xf8, 0xcf, 0x43, 0x5a, 0xd4, 0xc4, 0x33, 0xe9, 0x0e, 0x7b, 0xbb,
0xbf, 0xdb, 0xfd, 0xdd, 0xee, 0xde, 0x02, 0x20, 0x4d, 0x08, 0x5b, 0x4f, 0x05, 0x97, 0xdc, 0xbd,
0xd4, 0x8d, 0x93, 0x5e, 0x9e, 0xe9, 0xaf, 0x75, 0xb5, 0x71, 0xa5, 0x91, 0x85, 0x1d, 0xda, 0x25,
0x5a, 0xe4, 0x7d, 0x67, 0x40, 0x63, 0x8f, 0x32, 0x2a, 0xe2, 0xf0, 0x84, 0x24, 0x39, 0x75, 0xaf,
0x82, 0xd5, 0xe6, 0x3c, 0x09, 0x7a, 0x24, 0x59, 0x36, 0x56, 0x8d, 0x35, 0x6b, 0xbf, 0xe4, 0xd7,
0x94, 0xe4, 0x84, 0x24, 0xee, 0x35, 0xb0, 0x63, 0x26, 0xef, 0xdf, 0xc3, 0xdd, 0xf2, 0xaa, 0xb1,
0x66, 0xee, 0x97, 0x7c, 0x0b, 0x45, 0xc5, 0xf6, 0x69, 0xc2, 0x89, 0xc4, 0x6d, 0x73, 0xd5, 0x58,
0x33, 0xd4, 0x36, 0x8a, 0xd4, 0xf6, 0x0a, 0x40, 0x26, 0x45, 0xcc, 0xce, 0x70, 0xbf, 0xb2, 0x6a,
0xac, 0xd9, 0xfb, 0x25, 0xdf, 0xd6, 0xb2, 0x13, 0x92, 0x6c, 0x57, 0xc1, 0xec, 0x91, 0xc4, 0xfb,
0xd6, 0x00, 0xfb, 0xab, 0x9c, 0x8a, 0x7e, 0x8b, 0x9d, 0x72, 0xd7, 0x85, 0x8a, 0xe4, 0xe9, 0x4b,
0x74, 0xc6, 0xf4, 0x71, 0xed, 0xae, 0x40, 0xbd, 0x4b, 0xa5, 0x88, 0xc3, 0x40, 0xf6, 0x53, 0x8a,
0x47, 0xd9, 0x3e, 0x68, 0xd1, 0x71, 0x3f, 0xa5, 0xee, 0xff, 0x61, 0x31, 0xa3, 0x44, 0x84, 0x9d,
0x20, 0x25, 0x82, 0x74, 0x33, 0x7d, 0x9a, 0xdf, 0xd0, 0xc2, 0x23, 0x94, 0x29, 0x25, 0xc1, 0x73,
0x16, 0x05, 0x11, 0x0d, 0xe3, 0x2e, 0x49, 0x96, 0xab, 0x78, 0x44, 0x03, 0x85, 0xbb, 0x5a, 0xe6,
0xfd, 0x6a, 0x00, 0xec, 0xf0, 0x24, 0xef, 0x32, 0xf4, 0xe6, 0x32, 0x58, 0xa7, 0x31, 0x4d, 0xa2,
0x20, 0x8e, 0x0a, 0x8f, 0x6a, 0xf8, 0xdd, 0x8a, 0xdc, 0x47, 0x60, 0x47, 0x44, 0x12, 0xed, 0x92,
0x22, 0xa7, 0xf9, 0xf1, 0xb5, 0xf5, 0x09, 0xfe, 0x0b, 0xe6, 0x77, 0x89, 0x24, 0xca, 0x4b, 0xdf,
0x8a, 0x8a, 0x95, 0x7b, 0x03, 0x9a, 0x71, 0x16, 0xa4, 0x22, 0xee, 0x12, 0xd1, 0x0f, 0x5e, 0xd2,
0x3e, 0xc6, 0x64, 0xf9, 0x8d, 0x38, 0x3b, 0xd2, 0xc2, 0x2f, 0x69, 0xdf, 0xbd, 0x0a, 0x76, 0x9c,
0x05, 0x24, 0x97, 0xbc, 0xb5, 0x8b, 0x11, 0x59, 0xbe, 0x15, 0x67, 0x5b, 0xf8, 0xad, 0x38, 0x61,
0x34, 0x93, 0x34, 0x0a, 0x52, 0x22, 0x3b, 0xcb, 0xd5, 0x55, 0x53, 0x71, 0xa2, 0x45, 0x47, 0x44,
0x76, 0xbc, 0xcf, 0x07, 0x81, 0x3c, 0x79, 0x93, 0x0a, 0xf7, 0x0e, 0x54, 0x62, 0x76, 0xca, 0x31,
0x88, 0xfa, 0xdb, 0x8e, 0x62, 0x06, 0x8d, 0xa2, 0xf6, 0x51, 0xd5, 0xdb, 0x06, 0x1b, 0x73, 0x04,
0xed, 0x3f, 0x81, 0x6a, 0x4f, 0x7d, 0x14, 0x00, 0x2b, 0x53, 0x00, 0xc6, 0xf3, 0xca, 0xd7, 0xda,
0xde, 0x8f, 0x06, 0x34, 0x5f, 0x30, 0x22, 0xfa, 0x3e, 0x61, 0x67, 0x1a, 0xe9, 0x33, 0xa8, 0x87,
0x78, 0x54, 0x30, 0xbf, 0x43, 0x10, 0x8e, 0xae, 0xe4, 0x43, 0x28, 0xf3, 0xb4, 0x20, 0xfc, 0xf2,
0x14, 0xb3, 0xc3, 0x14, 0xc9, 0x2e, 0xf3, 0x74, 0xe4, 0xb4, 0x79, 0x21, 0xa7, 0x7f, 0x28, 0xc3,
0xd2, 0x76, 0xfc, 0x7e, 0xbd, 0xbe, 0x09, 0x4b, 0x09, 0x7f, 0x4d, 0x45, 0x10, 0xb3, 0x30, 0xc9,
0xb3, 0xb8, 0xa7, 0x73, 0xc6, 0xf2, 0x9b, 0x28, 0x6e, 0x0d, 0xa4, 0x4a, 0x31, 0x4f, 0xd3, 0x09,
0x45, 0x9d, 0x1b, 0x4d, 0x14, 0x8f, 0x14, 0x37, 0xa1, 0xae, 0x11, 0x75, 0x88, 0x95, 0xf9, 0x42,
0x04, 0xb4, 0xd1, 0xb5, 0xbf, 0x09, 0x75, 0x7d, 0x94, 0x46, 0xa8, 0xce, 0x89, 0x80, 0x36, 0xb8,
0xf6, 0x7e, 0x33, 0xa0, 0xbe, 0xc3, 0xbb, 0x29, 0x11, 0x9a, 0xa5, 0x3d, 0x70, 0x12, 0x7a, 0x2a,
0x83, 0x0b, 0x53, 0xd5, 0x54, 0x66, 0x63, 0x75, 0xd7, 0x82, 0x4b, 0x22, 0x3e, 0xeb, 0x4c, 0x22,
0x95, 0xe7, 0x41, 0x5a, 0x42, 0xbb, 0x9d, 0xb7, 0xf3, 0xc5, 0x9c, 0x23, 0x5f, 0xbc, 0x6f, 0x0c,
0xb0, 0x8e, 0xa9, 0xe8, 0xbe, 0x97, 0x1b, 0x7f, 0x00, 0x0b, 0xc8, 0x6b, 0xb6, 0x5c, 0x5e, 0x35,
0xe7, 0x21, 0xb6, 0x50, 0x57, 0x3d, 0xda, 0xc6, 0x9a, 0x41, 0x37, 0xee, 0xa1, 0xfb, 0x06, 0xba,
0x7f, 0x63, 0x0a, 0xc4, 0x50, 0x53, 0xaf, 0x0e, 0x53, 0xcc, 0xfc, 0xdb, 0x50, 0x0d, 0x3b, 0x71,
0x12, 0x15, 0x9c, 0xfd, 0x67, 0x8a, 0xa1, 0xb2, 0xf1, 0xb5, 0x96, 0xb7, 0x02, 0xb5, 0xc2, 0xda,
0xad, 0x43, 0xad, 0xc5, 0x7a, 0x24, 0x89, 0x23, 0xa7, 0xe4, 0xd6, 0xc0, 0x3c, 0xe0, 0xd2, 0x31,
0xbc, 0x3f, 0x0c, 0x00, 0x5d, 0x12, 0xe8, 0xd4, 0xfd, 0x31, 0xa7, 0x3e, 0x98, 0x82, 0x3d, 0x52,
0x2d, 0x96, 0x85, 0x5b, 0x1f, 0x41, 0x45, 0x5d, 0xf4, 0x79, 0x5e, 0xa1, 0x92, 0x8a, 0x01, 0xef,
0xb2, 0xa8, 0xde, 0xd9, 0x31, 0xa0, 0x96, 0x77, 0x1f, 0xac, 0xc1, 0x59, 0x93, 0x41, 0x34, 0x01,
0x9e, 0xf2, 0xb3, 0x38, 0x24, 0xc9, 0x16, 0x8b, 0x1c, 0xc3, 0x5d, 0x04, 0xbb, 0xf8, 0x3e, 0x14,
0x4e, 0xd9, 0xfb, 0xdd, 0x80, 0x45, 0x6d, 0xb8, 0x25, 0x62, 0xd9, 0x39, 0x4c, 0xff, 0xf1, 0xcd,
0x3f, 0x04, 0x8b, 0x28, 0xa8, 0x60, 0xd8, 0xa7, 0xae, 0x4f, 0x31, 0x2e, 0x4e, 0xc3, 0xe4, 0xab,
0x91, 0xe2, 0xe8, 0x5d, 0x58, 0xd4, 0x79, 0xcf, 0x53, 0x2a, 0x08, 0x8b, 0xe6, 0xed, 0x5c, 0x0d,
0xb4, 0x3a, 0xd4, 0x46, 0xde, 0xf7, 0xc6, 0xa0, 0x81, 0xe1, 0x21, 0x78, 0x65, 0x03, 0xea, 0x8d,
0x0b, 0x51, 0x5f, 0x9e, 0x87, 0x7a, 0x77, 0x7d, 0xac, 0xc4, 0xce, 0x0b, 0x55, 0xd5, 0xd9, 0x2f,
0x65, 0xb8, 0x32, 0x41, 0xf9, 0x93, 0x1e, 0x49, 0xde, 0x5f, 0xaf, 0xfd, 0xb7, 0xf9, 0x2f, 0x5a,
0x4e, 0xe5, 0x42, 0x4f, 0x54, 0xf5, 0x42, 0x4f, 0xd4, 0x5f, 0x55, 0xa8, 0x20, 0x57, 0x8f, 0xc0,
0x96, 0x54, 0x74, 0x03, 0xfa, 0x26, 0x15, 0x05, 0x53, 0x57, 0xa7, 0x60, 0x0c, 0xba, 0x9a, 0x1a,
0xd0, 0xe4, 0xa0, 0xc3, 0x3d, 0x06, 0xc8, 0xd5, 0x25, 0x68, 0x63, 0x7d, 0xd5, 0xff, 0x7d, 0x57,
0x8b, 0x51, 0xe3, 0x5b, 0x3e, 0x6c, 0x02, 0x9b, 0x50, 0x6f, 0xc7, 0x23, 0x7b, 0x73, 0xe6, 0x35,
0x8d, 0xba, 0xc1, 0x7e, 0xc9, 0x87, 0xf6, 0xa8, 0x8d, 0xec, 0x40, 0x23, 0xd4, 0xaf, 0x87, 0x86,
0xd0, 0x6f, 0xd8, 0xf5, 0xa9, 0x37, 0x3d, 0x7c, 0x64, 0xf6, 0x4b, 0x7e, 0x3d, 0x1c, 0x7b, 0x73,
0x9e, 0x81, 0xa3, 0xa3, 0x10, 0x2a, 0x81, 0x34, 0x90, 0x26, 0xf3, 0x7f, 0xb3, 0x62, 0x19, 0xa6,
0xda, 0x7e, 0xc9, 0x6f, 0xe6, 0x93, 0x0f, 0xfd, 0x11, 0x5c, 0x2a, 0xa2, 0x1a, 0xc3, 0x5b, 0x40,
0x3c, 0x6f, 0x66, 0x6c, 0xe3, 0x80, 0x4b, 0xed, 0xb7, 0x46, 0x07, 0x09, 0x2b, 0x05, 0xe2, 0x20,
0x2b, 0x03, 0xda, 0x23, 0xc9, 0x38, 0x7e, 0x0d, 0xf1, 0x6f, 0xcf, 0xc4, 0x9f, 0x56, 0x26, 0xfb,
0x25, 0xff, 0x4a, 0x7b, 0x76, 0x11, 0x8d, 0xe2, 0xd0, 0xa7, 0xe2, 0x39, 0xd6, 0x39, 0x71, 0x0c,
0xdb, 0xc5, 0x28, 0x8e, 0x51, 0x07, 0x79, 0x0c, 0x80, 0xc9, 0xa7, 0xa1, 0xec, 0x99, 0xe9, 0x32,
0x1c, 0x1a, 0x55, 0xba, 0xf4, 0x86, 0x13, 0xe4, 0xe6, 0xb0, 0xaa, 0xd1, 0x1e, 0xce, 0xa9, 0xea,
0x41, 0xba, 0x84, 0xc3, 0xaf, 0xed, 0x05, 0xa8, 0x28, 0x53, 0xef, 0x4f, 0x03, 0xe0, 0x84, 0x86,
0x92, 0x8b, 0xad, 0x83, 0x83, 0xe7, 0xc5, 0x98, 0xac, 0xbd, 0xd5, 0xff, 0x30, 0x6a, 0x4c, 0xd6,
0x01, 0x4d, 0x0c, 0xf0, 0xe5, 0xc9, 0x01, 0xfe, 0x01, 0x40, 0x2a, 0x68, 0x14, 0x87, 0x44, 0xd2,
0xec, 0xbc, 0x47, 0x66, 0x4c, 0xd5, 0xfd, 0x14, 0xe0, 0x95, 0xfa, 0x5f, 0xd1, 0xed, 0xa9, 0x32,
0x93, 0x88, 0xe1, 0x4f, 0x8d, 0x6f, 0xbf, 0x1a, 0xfe, 0xdf, 0xdc, 0x84, 0xa5, 0x34, 0x21, 0x21,
0xed, 0xf0, 0x24, 0xa2, 0x22, 0x90, 0xe4, 0x0c, 0xb3, 0xd5, 0xf6, 0x9b, 0x63, 0xe2, 0x63, 0x72,
0xe6, 0xfd, 0x64, 0x80, 0x75, 0x94, 0x10, 0x76, 0xc0, 0x23, 0x1c, 0xd5, 0x7a, 0x18, 0x71, 0x40,
0x18, 0xcb, 0xde, 0xd1, 0x12, 0x47, 0xbc, 0x28, 0xf2, 0xb4, 0xcd, 0x16, 0x63, 0x99, 0xfb, 0x70,
0x22, 0xda, 0x77, 0xf7, 0x75, 0x65, 0x3a, 0x16, 0xef, 0x1a, 0x38, 0x3c, 0x97, 0x69, 0x2e, 0x83,
0x01, 0x95, 0x8a, 0x2e, 0x73, 0xcd, 0xf4, 0x9b, 0x5a, 0xfe, 0x85, 0x66, 0x34, 0x53, 0x37, 0xc4,
0x78, 0x44, 0x6f, 0xfd, 0x6c, 0xc0, 0x82, 0x6e, 0x72, 0x93, 0x4f, 0xf1, 0x12, 0xd4, 0xf7, 0x04,
0x25, 0x92, 0x8a, 0xe3, 0x0e, 0x61, 0x8e, 0xe1, 0x3a, 0xd0, 0x28, 0x04, 0x4f, 0x5e, 0xe5, 0x24,
0x71, 0xca, 0x6e, 0x03, 0xac, 0xa7, 0x34, 0xcb, 0x70, 0xdf, 0xc4, 0xb7, 0x9a, 0x66, 0x99, 0xde,
0xac, 0xb8, 0x36, 0x54, 0xf5, 0xb2, 0xaa, 0xf4, 0x0e, 0xb8, 0xd4, 0x5f, 0x0b, 0x0a, 0xf8, 0x48,
0xd0, 0xd3, 0xf8, 0xcd, 0x33, 0x22, 0xc3, 0x8e, 0x53, 0x53, 0xc0, 0x47, 0x3c, 0x93, 0x43, 0x89,
0xa5, 0x6c, 0xf5, 0xd2, 0x56, 0x4b, 0x2c, 0x14, 0x07, 0xdc, 0x05, 0x28, 0xb7, 0x98, 0x53, 0x57,
0xa2, 0x03, 0x2e, 0x5b, 0xcc, 0x69, 0xdc, 0xda, 0x83, 0xfa, 0xd8, 0xdb, 0xa0, 0x02, 0x78, 0xc1,
0x5e, 0x32, 0xfe, 0x9a, 0xe9, 0x81, 0x68, 0x2b, 0x52, 0x43, 0x44, 0x0d, 0xcc, 0xe7, 0x79, 0xdb,
0x29, 0xab, 0xc5, 0xb3, 0x3c, 0x71, 0x4c, 0xb5, 0xd8, 0x8d, 0x7b, 0x4e, 0x05, 0x25, 0x3c, 0x72,
0xaa, 0xdb, 0x77, 0xbf, 0xbe, 0x73, 0x16, 0xcb, 0x4e, 0xde, 0x5e, 0x0f, 0x79, 0x77, 0x43, 0x53,
0x7d, 0x3b, 0xe6, 0xc5, 0x6a, 0x23, 0x66, 0x92, 0x0a, 0x46, 0x92, 0x0d, 0x64, 0x7f, 0x43, 0xb1,
0x9f, 0xb6, 0xdb, 0x0b, 0xf8, 0x75, 0xf7, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2d, 0x26, 0x94,
0x29, 0xd7, 0x0f, 0x00, 0x00,
}

5
internal/proxy/impl.go
View File

@ -2543,8 +2543,7 @@ func (node *Proxy) Insert(ctx context.Context, request *milvuspb.InsertRequest)
it := &insertTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
// req: request,
BaseInsertTask: BaseInsertTask{
BaseInsertTask: msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
HashValues: request.HashKeys,
},
@ -5307,7 +5306,7 @@ func (node *Proxy) CreateDatabase(ctx context.Context, request *milvuspb.CreateD
panic("implement me")
}
func (node *Proxy) DropDatabase(ctx context.Context, request *milvuspb.CreateDatabaseRequest) (*commonpb.Status, error) {
func (node *Proxy) DropDatabase(ctx context.Context, request *milvuspb.DropDatabaseRequest) (*commonpb.Status, error) {
//TODO implement me
panic("implement me")
}

1
internal/querynode/plan_test.go
View File

@ -139,7 +139,6 @@ func TestPlan_newSearchRequest(t *testing.T) {
searchReq, err := newSearchRequest(collection, req, req.Req.GetPlaceholderGroup())
assert.NoError(t, err)
assert.Equal(t, simpleFloatVecField.id, searchReq.searchFieldID)
assert.EqualValues(t, defaultNQ, searchReq.getNumOfQuery())
searchReq.delete()

145
internal/storage/data_codec.go
View File

@ -133,6 +133,13 @@ type DoubleFieldData struct {
type StringFieldData struct {
Data []string
}
type ArrayFieldData struct {
ElementType schemapb.DataType
Data []*schemapb.ScalarField
}
type JSONFieldData struct {
Data [][]byte
}
type BinaryVectorFieldData struct {
Data []byte
Dim int
@ -153,20 +160,24 @@ func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
func (data *StringFieldData) RowNum() int { return len(data.Data) }
func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim }
func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim }
func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
// GetRow implements FieldData.GetRow
func (data *BoolFieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *Int8FieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *Int16FieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *Int32FieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *Int64FieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *FloatFieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *DoubleFieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *StringFieldData) GetRow(i int) interface{} { return data.Data[i] }
func (data *BinaryVectorFieldData) GetRow(i int) interface{} {
func (data *BoolFieldData) GetRow(i int) any { return data.Data[i] }
func (data *Int8FieldData) GetRow(i int) any { return data.Data[i] }
func (data *Int16FieldData) GetRow(i int) any { return data.Data[i] }
func (data *Int32FieldData) GetRow(i int) any { return data.Data[i] }
func (data *Int64FieldData) GetRow(i int) any { return data.Data[i] }
func (data *FloatFieldData) GetRow(i int) any { return data.Data[i] }
func (data *DoubleFieldData) GetRow(i int) any { return data.Data[i] }
func (data *StringFieldData) GetRow(i int) any { return data.Data[i] }
func (data *ArrayFieldData) GetRow(i int) any { return data.Data[i] }
func (data *JSONFieldData) GetRow(i int) any { return data.Data[i] }
func (data *BinaryVectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim/8 : (i+1)*data.Dim/8]
}
func (data *FloatVectorFieldData) GetRow(i int) interface{} {
func (data *FloatVectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim : (i+1)*data.Dim]
}
@ -216,6 +227,37 @@ func (data *StringFieldData) GetMemorySize() int {
return size
}
func (data *ArrayFieldData) GetMemorySize() int {
var size int
for _, val := range data.Data {
switch data.ElementType {
case schemapb.DataType_Bool:
size += binary.Size(val.GetBoolData().GetData())
case schemapb.DataType_Int8:
size += binary.Size(val.GetIntData().GetData()) / 4
case schemapb.DataType_Int16:
size += binary.Size(val.GetIntData().GetData()) / 2
case schemapb.DataType_Int32:
size += binary.Size(val.GetIntData().GetData())
case schemapb.DataType_Float:
size += binary.Size(val.GetFloatData().GetData())
case schemapb.DataType_Double:
size += binary.Size(val.GetDoubleData().GetData())
case schemapb.DataType_String, schemapb.DataType_VarChar:
size += (&StringFieldData{Data: val.GetStringData().GetData()}).GetMemorySize()
}
}
return size
}
func (data *JSONFieldData) GetMemorySize() int {
var size int
for _, val := range data.Data {
size += len(val) + 16
}
return size
}
func (data *BinaryVectorFieldData) GetMemorySize() int {
return binary.Size(data.Data) + 4
}
@ -386,6 +428,26 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique
}
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*StringFieldData).GetMemorySize()))
case schemapb.DataType_Array:
for _, singleArray := range singleData.(*ArrayFieldData).Data {
err = eventWriter.AddOneArrayToPayload(singleArray)
if err != nil {
eventWriter.Close()
writer.Close()
return nil, nil, err
}
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*ArrayFieldData).GetMemorySize()))
case schemapb.DataType_JSON:
for _, singleJSON := range singleData.(*JSONFieldData).Data {
err = eventWriter.AddOneJSONToPayload(singleJSON)
if err != nil {
eventWriter.Close()
writer.Close()
return nil, nil, err
}
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*JSONFieldData).GetMemorySize()))
case schemapb.DataType_BinaryVector:
err = eventWriter.AddBinaryVectorToPayload(singleData.(*BinaryVectorFieldData).Data, singleData.(*BinaryVectorFieldData).Dim)
if err != nil {
@ -656,6 +718,44 @@ func (insertCodec *InsertCodec) DeserializeInto(fieldBinlogs []*Blob, rowNum int
totalLength += len(stringPayload)
insertData.Data[fieldID] = stringFieldData
case schemapb.DataType_Array:
arrayPayload, err := eventReader.GetArrayFromPayload()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
if insertData.Data[fieldID] == nil {
insertData.Data[fieldID] = &ArrayFieldData{
Data: make([]*schemapb.ScalarField, 0, rowNum),
}
}
arrayFieldData := insertData.Data[fieldID].(*ArrayFieldData)
arrayFieldData.Data = append(arrayFieldData.Data, arrayPayload...)
totalLength += len(arrayPayload)
insertData.Data[fieldID] = arrayFieldData
case schemapb.DataType_JSON:
jsonPayload, err := eventReader.GetJSONFromPayload()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
if insertData.Data[fieldID] == nil {
insertData.Data[fieldID] = &JSONFieldData{
Data: make([][]byte, 0, rowNum),
}
}
jsonFieldData := insertData.Data[fieldID].(*JSONFieldData)
jsonFieldData.Data = append(jsonFieldData.Data, jsonPayload...)
totalLength += len(jsonPayload)
insertData.Data[fieldID] = jsonFieldData
case schemapb.DataType_BinaryVector:
var singleData []byte
singleData, dim, err = eventReader.GetBinaryVectorFromPayload()
@ -734,6 +834,31 @@ func (insertCodec *InsertCodec) DeserializeInto(fieldBinlogs []*Blob, rowNum int
return collectionID, partitionID, segmentID, nil
}
// func deserializeEntity[T any, U any](
// eventReader *EventReader,
// binlogReader *BinlogReader,
// insertData *InsertData,
// getPayloadFunc func() (U, error),
// fillDataFunc func() FieldData,
// ) error {
// fieldID := binlogReader.FieldID
// stringPayload, err := getPayloadFunc()
// if err != nil {
// eventReader.Close()
// binlogReader.Close()
// return err
// }
//
// if insertData.Data[fieldID] == nil {
// insertData.Data[fieldID] = fillDataFunc()
// }
// stringFieldData := insertData.Data[fieldID].(*T)
//
// stringFieldData.Data = append(stringFieldData.Data, stringPayload...)
// totalLength += len(stringPayload)
// insertData.Data[fieldID] = stringFieldData
// }
// Deserialize transfer blob back to insert data.
// From schema, it get all fields.
// For each field, it will create a binlog reader, and read all event to the buffer.

93
internal/storage/data_codec_test.go
View File

@ -45,6 +45,8 @@ const (
StringField = 107
BinaryVectorField = 108
FloatVectorField = 109
ArrayField = 110
JSONField = 111
)
func TestInsertCodec(t *testing.T) {
@ -128,6 +130,19 @@ func TestInsertCodec(t *testing.T) {
Description: "string",
DataType: schemapb.DataType_String,
},
{
FieldID: ArrayField,
Name: "field_int32_array",
Description: "int32 array",
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int32,
},
{
FieldID: JSONField,
Name: "field_json",
Description: "json",
DataType: schemapb.DataType_JSON,
},
{
FieldID: BinaryVectorField,
Name: "field_binary_vector",
@ -186,6 +201,27 @@ func TestInsertCodec(t *testing.T) {
Data: []float32{4, 5, 6, 7, 4, 5, 6, 7},
Dim: 4,
},
ArrayField: &ArrayFieldData{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: []int32{3, 2, 1}},
},
},
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: []int32{6, 5, 4}},
},
},
},
},
JSONField: &JSONFieldData{
Data: [][]byte{
[]byte(`{"batch":2}`),
[]byte(`{"key":"world"}`),
},
},
},
}
@ -229,6 +265,27 @@ func TestInsertCodec(t *testing.T) {
Data: []float32{0, 1, 2, 3, 0, 1, 2, 3},
Dim: 4,
},
ArrayField: &ArrayFieldData{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: []int32{1, 2, 3}},
},
},
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: []int32{4, 5, 6}},
},
},
},
},
JSONField: &JSONFieldData{
Data: [][]byte{
[]byte(`{"batch":1}`),
[]byte(`{"key":"hello"}`),
},
},
},
}
@ -246,6 +303,8 @@ func TestInsertCodec(t *testing.T) {
StringField: &StringFieldData{[]string{}},
BinaryVectorField: &BinaryVectorFieldData{[]byte{}, 8},
FloatVectorField: &FloatVectorFieldData{[]float32{}, 4},
ArrayField: &ArrayFieldData{schemapb.DataType_Int32, []*schemapb.ScalarField{}},
JSONField: &JSONFieldData{[][]byte{}},
},
}
b, s, err := insertCodec.Serialize(PartitionID, SegmentID, insertDataEmpty)
@ -283,6 +342,23 @@ func TestInsertCodec(t *testing.T) {
assert.Equal(t, []string{"1", "2", "3", "4"}, resultData.Data[StringField].(*StringFieldData).Data)
assert.Equal(t, []byte{0, 255, 0, 255}, resultData.Data[BinaryVectorField].(*BinaryVectorFieldData).Data)
assert.Equal(t, []float32{0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}, resultData.Data[FloatVectorField].(*FloatVectorFieldData).Data)
int32ArrayList := [][]int32{{1, 2, 3}, {4, 5, 6}, {3, 2, 1}, {6, 5, 4}}
resultArrayList := [][]int32{}
for _, v := range resultData.Data[ArrayField].(*ArrayFieldData).Data {
resultArrayList = append(resultArrayList, v.GetIntData().GetData())
}
assert.EqualValues(t, int32ArrayList, resultArrayList)
assert.Equal(t,
[][]byte{
[]byte(`{"batch":1}`),
[]byte(`{"key":"hello"}`),
[]byte(`{"batch":2}`),
[]byte(`{"key":"world"}`),
},
resultData.Data[JSONField].(*JSONFieldData).Data)
log.Debug("Data", zap.Any("Data", resultData.Data))
log.Debug("Infos", zap.Any("Infos", resultData.Infos))
@ -465,6 +541,21 @@ func TestMemorySize(t *testing.T) {
Data: []float32{4, 5, 6, 7},
Dim: 4,
},
ArrayField: &ArrayFieldData{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{Data: []int32{1, 2, 3}},
},
},
},
},
JSONField: &JSONFieldData{
Data: [][]byte{
[]byte(`{"batch":1}`),
},
},
},
}
assert.Equal(t, insertData1.Data[RowIDField].GetMemorySize(), 8)
@ -479,6 +570,8 @@ func TestMemorySize(t *testing.T) {
assert.Equal(t, insertData1.Data[StringField].GetMemorySize(), 17)
assert.Equal(t, insertData1.Data[BinaryVectorField].GetMemorySize(), 5)
assert.Equal(t, insertData1.Data[FloatField].GetMemorySize(), 4)
assert.Equal(t, insertData1.Data[ArrayField].GetMemorySize(), 3*4)
assert.Equal(t, insertData1.Data[JSONField].GetMemorySize(), len([]byte(`{"batch":1}`))+16)
insertData2 := &InsertData{
Data: map[int64]FieldData{

6
internal/storage/data_sorter.go
View File

@ -94,6 +94,12 @@ func (ds *DataSorter) Swap(i, j int) {
for idx := 0; idx < dim; idx++ {
data[i*dim+idx], data[j*dim+idx] = data[j*dim+idx], data[i*dim+idx]
}
case schemapb.DataType_Array:
data := singleData.(*ArrayFieldData).Data
data[i], data[j] = data[j], data[i]
case schemapb.DataType_JSON:
data := singleData.(*JSONFieldData).Data
data[i], data[j] = data[j], data[i]
default:
errMsg := "undefined data type " + string(field.DataType)
panic(errMsg)

42
internal/storage/payload.go
View File

@ -29,6 +29,7 @@ import (
"reflect"
"unsafe"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -45,6 +46,8 @@ type PayloadWriterInterface interface {
AddFloatToPayload(msgs []float32) error
AddDoubleToPayload(msgs []float64) error
AddOneStringToPayload(msgs string) error
AddOneArrayToPayload(msg *schemapb.ScalarField) error
AddOneJSONToPayload(msg []byte) error
AddBinaryVectorToPayload(binVec []byte, dim int) error
AddFloatVectorToPayload(binVec []float32, dim int) error
FinishPayloadWriter() error
@ -66,6 +69,8 @@ type PayloadReaderInterface interface {
GetFloatFromPayload() ([]float32, error)
GetDoubleFromPayload() ([]float64, error)
GetStringFromPayload() ([]string, error)
GetArrayFromPayload() ([]*schemapb.ScalarField, error)
GetJSONFromPayload() ([][]byte, error)
GetBinaryVectorFromPayload() ([]byte, int, error)
GetFloatVectorFromPayload() ([]float32, int, error)
GetPayloadLengthFromReader() (int, error)
@ -149,6 +154,18 @@ func (w *PayloadWriter) AddDataToPayload(msgs interface{}, dim ...int) error {
return errors.New("incorrect data type")
}
return w.AddOneStringToPayload(val)
case schemapb.DataType_Array:
val, ok := msgs.(*schemapb.ScalarField)
if !ok {
return errors.New("incorrect data type")
}
return w.AddOneArrayToPayload(val)
case schemapb.DataType_JSON:
val, ok := msgs.([]byte)
if !ok {
return errors.New("incorrect data type")
}
return w.AddOneJSONToPayload(val)
default:
return errors.New("incorrect datatype")
}
@ -289,6 +306,31 @@ func (w *PayloadWriter) AddOneStringToPayload(msg string) error {
return HandleCStatus(&status, "AddOneStringToPayload failed")
}
func (w *PayloadWriter) AddOneArrayToPayload(msg *schemapb.ScalarField) error {
bytes, err := proto.Marshal(msg)
if err != nil {
return errors.New("Marshal ListValue failed")
}
length := len(bytes)
cmsg := (*C.uint8_t)(unsafe.Pointer(&bytes[0]))
clength := C.int(length)
// defer C.free(unsafe.Pointer(cmsg))
status := C.AddOneArrayToPayload(w.payloadWriterPtr, cmsg, clength)
return HandleCStatus(&status, "AddOneArrayToPayload failed")
}
func (w *PayloadWriter) AddOneJSONToPayload(msg []byte) error {
bytes := msg
length := len(bytes)
cmsg := (*C.uint8_t)(unsafe.Pointer(&bytes[0]))
clength := C.int(length)
status := C.AddOneJSONToPayload(w.payloadWriterPtr, cmsg, clength)
return HandleCStatus(&status, "AddOneJSONToPayload failed")
}
// AddBinaryVectorToPayload dimension > 0 && (%8 == 0)
func (w *PayloadWriter) AddBinaryVectorToPayload(binVec []byte, dim int) error {
length := len(binVec)

221
internal/storage/payload_benchmark_test.go
View File

@ -1,221 +0,0 @@
package storage
import (
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
)
// workload setting for benchmark
const (
numElements = 10000
vectorDim = 512
)
func BenchmarkPayloadReader_Bool(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_Bool)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]bool, 0, numElements)
for i := 0; i < numElements; i++ {
data = append(data, rand.Intn(2) != 0)
}
w.AddBoolToPayload(data)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_Bool, buffer)
r.GetBoolFromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_Bool, buffer)
r.GetBoolFromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_Int32(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_Int32)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]int32, 0, numElements)
for i := 0; i < numElements; i++ {
data = append(data, rand.Int31n(1000))
}
w.AddInt32ToPayload(data)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_Int32, buffer)
r.GetInt32FromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_Int32, buffer)
r.GetInt32FromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_Int64(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_Int64)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]int64, 0, numElements)
for i := 0; i < numElements; i++ {
data = append(data, rand.Int63n(1000))
}
w.AddInt64ToPayload(data)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_Int64, buffer)
r.GetInt64FromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_Int64, buffer)
r.GetInt64FromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_Float32(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_Float)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]float32, 0, numElements)
for i := 0; i < numElements; i++ {
data = append(data, rand.Float32())
}
w.AddFloatToPayload(data)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_Float, buffer)
r.GetFloatFromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_Float, buffer)
r.GetFloatFromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_Float64(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_Double)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]float64, 0, numElements)
for i := 0; i < numElements; i++ {
data = append(data, rand.Float64())
}
w.AddDoubleToPayload(data)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_Double, buffer)
r.GetDoubleFromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_Double, buffer)
r.GetDoubleFromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_FloatVector(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_FloatVector, vectorDim)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]float32, 0, numElements*vectorDim)
for i := 0; i < numElements; i++ {
data = append(data, rand.Float32())
}
w.AddFloatVectorToPayload(data, vectorDim)
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_FloatVector, buffer)
r.GetFloatVectorFromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_FloatVector, buffer)
r.GetFloatVectorFromPayload()
r.ReleasePayloadReader()
}
})
}
func BenchmarkPayloadReader_BinaryVector(b *testing.B) {
w, err := NewPayloadWriter(schemapb.DataType_BinaryVector, vectorDim)
assert.NoError(b, err)
defer w.ReleasePayloadWriter()
data := make([]byte, numElements*vectorDim/8)
rand.Read(data)
err = w.AddBinaryVectorToPayload(data, vectorDim)
if err != nil {
panic(err)
}
w.FinishPayloadWriter()
buffer, _ := w.GetPayloadBufferFromWriter()
b.Run("cgo reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReaderCgo(schemapb.DataType_BinaryVector, buffer)
r.GetBinaryVectorFromPayload()
r.ReleasePayloadReader()
}
})
b.Run("go reader", func(b *testing.B) {
for i := 0; i < b.N; i++ {
r, _ := NewPayloadReader(schemapb.DataType_BinaryVector, buffer)
r.GetBinaryVectorFromPayload()
r.ReleasePayloadReader()
}
})
}

40
internal/storage/payload_reader.go
View File

@ -8,6 +8,7 @@ import (
"github.com/apache/arrow/go/v8/arrow"
"github.com/apache/arrow/go/v8/parquet"
"github.com/apache/arrow/go/v8/parquet/file"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
)
@ -19,6 +20,8 @@ type PayloadReader struct {
numRows int64
}
var _ PayloadReaderInterface = (*PayloadReader)(nil)
func NewPayloadReader(colType schemapb.DataType, buf []byte) (*PayloadReader, error) {
if len(buf) == 0 {
return nil, errors.New("create Payload reader failed, buffer is empty")
@ -66,6 +69,12 @@ func (r *PayloadReader) GetDataFromPayload() (interface{}, int, error) {
case schemapb.DataType_String, schemapb.DataType_VarChar:
val, err := r.GetStringFromPayload()
return val, 0, err
case schemapb.DataType_Array:
val, err := r.GetArrayFromPayload()
return val, 0, err
case schemapb.DataType_JSON:
val, err := r.GetJSONFromPayload()
return val, 0, err
default:
return nil, 0, errors.New("unknown type")
}
@ -237,6 +246,33 @@ func (r *PayloadReader) GetStringFromPayload() ([]string, error) {
return nil, fmt.Errorf("failed to get string from datatype %v", r.colType.String())
}
return readByteAndConvert(r, func(bytes parquet.ByteArray) string {
return bytes.String()
})
}
func (r *PayloadReader) GetArrayFromPayload() ([]*schemapb.ScalarField, error) {
if r.colType != schemapb.DataType_Array {
return nil, fmt.Errorf("failed to get string from datatype %v", r.colType.String())
}
return readByteAndConvert(r, func(bytes parquet.ByteArray) *schemapb.ScalarField {
v := &schemapb.ScalarField{}
proto.Unmarshal(bytes, v)
return v
})
}
func (r *PayloadReader) GetJSONFromPayload() ([][]byte, error) {
if r.colType != schemapb.DataType_JSON {
return nil, fmt.Errorf("failed to get string from datatype %v", r.colType.String())
}
return readByteAndConvert(r, func(bytes parquet.ByteArray) []byte {
return bytes
})
}
func readByteAndConvert[T any](r *PayloadReader, convert func(parquet.ByteArray) T) ([]T, error) {
values := make([]parquet.ByteArray, r.numRows)
valuesRead, err := ReadDataFromAllRowGroups[parquet.ByteArray, *file.ByteArrayColumnChunkReader](r.reader, values, 0, r.numRows)
if err != nil {
@ -247,9 +283,9 @@ func (r *PayloadReader) GetStringFromPayload() ([]string, error) {
return nil, fmt.Errorf("expect %d rows, but got valuesRead = %d", r.numRows, valuesRead)
}
ret := make([]string, r.numRows)
ret := make([]T, r.numRows)
for i := 0; i < int(r.numRows); i++ {
ret[i] = values[i].String()
ret[i] = convert(values[i])
}
return ret, nil
}

116
internal/storage/payload_test.go
View File

@ -332,6 +332,122 @@ func TestPayload_ReaderAndWriter(t *testing.T) {
w.ReleasePayloadWriter()
})
t.Run("TestAddArray", func(t *testing.T) {
w, err := NewPayloadWriter(schemapb.DataType_Array)
require.Nil(t, err)
require.NotNil(t, w)
err = w.AddOneArrayToPayload(&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{1, 2},
},
},
})
assert.Nil(t, err)
err = w.AddOneArrayToPayload(&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{3, 4},
},
},
})
assert.Nil(t, err)
err = w.AddOneArrayToPayload(&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{5, 6},
},
},
})
assert.Nil(t, err)
err = w.AddDataToPayload(&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{7, 8},
},
},
})
assert.Nil(t, err)
err = w.FinishPayloadWriter()
assert.Nil(t, err)
length, err := w.GetPayloadLengthFromWriter()
assert.Nil(t, err)
assert.Equal(t, length, 4)
buffer, err := w.GetPayloadBufferFromWriter()
assert.Nil(t, err)
r, err := NewPayloadReader(schemapb.DataType_Array, buffer)
assert.Nil(t, err)
length, err = r.GetPayloadLengthFromReader()
assert.Nil(t, err)
assert.Equal(t, length, 4)
arrayList, err := r.GetArrayFromPayload()
assert.Nil(t, err)
assert.EqualValues(t, []int32{1, 2}, arrayList[0].GetIntData().GetData())
assert.EqualValues(t, []int32{3, 4}, arrayList[1].GetIntData().GetData())
assert.EqualValues(t, []int32{5, 6}, arrayList[2].GetIntData().GetData())
assert.EqualValues(t, []int32{7, 8}, arrayList[3].GetIntData().GetData())
iArrayList, _, err := r.GetDataFromPayload()
arrayList = iArrayList.([]*schemapb.ScalarField)
assert.Nil(t, err)
assert.EqualValues(t, []int32{1, 2}, arrayList[0].GetIntData().GetData())
assert.EqualValues(t, []int32{3, 4}, arrayList[1].GetIntData().GetData())
assert.EqualValues(t, []int32{5, 6}, arrayList[2].GetIntData().GetData())
assert.EqualValues(t, []int32{7, 8}, arrayList[3].GetIntData().GetData())
r.ReleasePayloadReader()
w.ReleasePayloadWriter()
})
t.Run("TestAddJSON", func(t *testing.T) {
w, err := NewPayloadWriter(schemapb.DataType_JSON)
require.Nil(t, err)
require.NotNil(t, w)
err = w.AddOneJSONToPayload([]byte(`{"1":"1"}`))
assert.Nil(t, err)
err = w.AddOneJSONToPayload([]byte(`{"2":"2"}`))
assert.Nil(t, err)
err = w.AddOneJSONToPayload([]byte(`{"3":"3"}`))
assert.Nil(t, err)
err = w.AddDataToPayload([]byte(`{"4":"4"}`))
assert.Nil(t, err)
err = w.FinishPayloadWriter()
assert.Nil(t, err)
length, err := w.GetPayloadLengthFromWriter()
assert.Nil(t, err)
assert.Equal(t, length, 4)
buffer, err := w.GetPayloadBufferFromWriter()
assert.Nil(t, err)
r, err := NewPayloadReader(schemapb.DataType_JSON, buffer)
assert.Nil(t, err)
length, err = r.GetPayloadLengthFromReader()
assert.Nil(t, err)
assert.Equal(t, length, 4)
json, err := r.GetJSONFromPayload()
assert.Nil(t, err)
assert.EqualValues(t, []byte(`{"1":"1"}`), json[0])
assert.EqualValues(t, []byte(`{"2":"2"}`), json[1])
assert.EqualValues(t, []byte(`{"3":"3"}`), json[2])
assert.EqualValues(t, []byte(`{"4":"4"}`), json[3])
iJSON, _, err := r.GetDataFromPayload()
json = iJSON.([][]byte)
assert.Nil(t, err)
assert.EqualValues(t, []byte(`{"1":"1"}`), json[0])
assert.EqualValues(t, []byte(`{"2":"2"}`), json[1])
assert.EqualValues(t, []byte(`{"3":"3"}`), json[2])
assert.EqualValues(t, []byte(`{"4":"4"}`), json[3])
r.ReleasePayloadReader()
w.ReleasePayloadWriter()
})
t.Run("TestBinaryVector", func(t *testing.T) {
w, err := NewPayloadWriter(schemapb.DataType_BinaryVector, 8)
require.Nil(t, err)

86
internal/storage/utils.go
View File

@ -529,6 +529,24 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
Data: make([]string, 0, len(srcData)),
}
fieldData.Data = append(fieldData.Data, srcData...)
idata.Data[field.FieldID] = fieldData
case schemapb.DataType_Array:
srcData := srcFields[field.FieldID].GetScalars().GetArrayData().GetData()
fieldData := &ArrayFieldData{
Data: make([]*schemapb.ScalarField, 0, len(srcData)),
}
fieldData.Data = append(fieldData.Data, srcData...)
idata.Data[field.FieldID] = fieldData
case schemapb.DataType_JSON:
srcData := srcFields[field.FieldID].GetScalars().GetJsonData().GetData()
fieldData := &JSONFieldData{
Data: make([][]byte, 0, len(srcData)),
}
fieldData.Data = append(fieldData.Data, srcData...)
idata.Data[field.FieldID] = fieldData
}
@ -632,6 +650,28 @@ func mergeStringField(data *InsertData, fid FieldID, field *StringFieldData) {
fieldData.Data = append(fieldData.Data, field.Data...)
}
func mergeArrayField(data *InsertData, fid FieldID, field *ArrayFieldData) {
if _, ok := data.Data[fid]; !ok {
fieldData := &ArrayFieldData{
Data: nil,
}
data.Data[fid] = fieldData
}
fieldData := data.Data[fid].(*ArrayFieldData)
fieldData.Data = append(fieldData.Data, field.Data...)
}
func mergeJSONField(data *InsertData, fid FieldID, field *JSONFieldData) {
if _, ok := data.Data[fid]; !ok {
fieldData := &JSONFieldData{
Data: nil,
}
data.Data[fid] = fieldData
}
fieldData := data.Data[fid].(*JSONFieldData)
fieldData.Data = append(fieldData.Data, field.Data...)
}
func mergeBinaryVectorField(data *InsertData, fid FieldID, field *BinaryVectorFieldData) {
if _, ok := data.Data[fid]; !ok {
fieldData := &BinaryVectorFieldData{
@ -678,6 +718,10 @@ func MergeFieldData(data *InsertData, fid FieldID, field FieldData) {
mergeDoubleField(data, fid, field)
case *StringFieldData:
mergeStringField(data, fid, field)
case *ArrayFieldData:
mergeArrayField(data, fid, field)
case *JSONFieldData:
mergeJSONField(data, fid, field)
case *BinaryVectorFieldData:
mergeBinaryVectorField(data, fid, field)
case *FloatVectorFieldData:
@ -769,6 +813,16 @@ func stringFieldDataToPbBytes(field *StringFieldData) ([]byte, error) {
return proto.Marshal(arr)
}
func arrayFieldDataToPbBytes(field *ArrayFieldData) ([]byte, error) {
arr := &schemapb.ArrayArray{Data: field.Data}
return proto.Marshal(arr)
}
func jsonFieldDataToPbBytes(field *JSONFieldData) ([]byte, error) {
arr := &schemapb.JSONArray{Data: field.Data}
return proto.Marshal(arr)
}
func binaryWrite(endian binary.ByteOrder, data interface{}) ([]byte, error) {
buf := new(bytes.Buffer)
err := binary.Write(buf, endian, data)
@ -791,6 +845,10 @@ func FieldDataToBytes(endian binary.ByteOrder, fieldData FieldData) ([]byte, err
return boolFieldDataToPbBytes(field)
case *StringFieldData:
return stringFieldDataToPbBytes(field)
case *ArrayFieldData:
return arrayFieldDataToPbBytes(field)
case *JSONFieldData:
return jsonFieldDataToPbBytes(field)
case *BinaryVectorFieldData:
return field.Data, nil
case *FloatVectorFieldData:
@ -937,6 +995,34 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
},
},
}
case *ArrayFieldData:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldId: fieldID,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: rawData.Data,
},
},
},
},
}
case *JSONFieldData:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_JSON,
FieldId: fieldID,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: rawData.Data,
},
},
},
},
}
case *FloatVectorFieldData:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_FloatVector,

129
internal/storage/utils_test.go
View File

@ -361,6 +361,12 @@ func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema
},
},
},
{
DataType: schemapb.DataType_Array,
},
{
DataType: schemapb.DataType_JSON,
},
},
}
fieldIDs = make([]UniqueID, 0)
@ -448,6 +454,28 @@ func generateFloat64Array(numRows int) []float64 {
return ret
}
func generateBytesArray(numRows int) [][]byte {
ret := make([][]byte, 0, numRows)
for i := 0; i < numRows; i++ {
ret = append(ret, []byte(fmt.Sprint(rand.Int())))
}
return ret
}
func generateInt32ArrayList(numRows int) []*schemapb.ScalarField {
ret := make([]*schemapb.ScalarField, 0, numRows)
for i := 0; i < numRows; i++ {
ret = append(ret, &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{rand.Int31(), rand.Int31()},
},
},
})
}
return ret
}
func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, row []interface{}) {
schema, _, _ := genAllFieldsSchema(fVecDim, bVecDim)
ret := &commonpb.Blob{
@ -502,6 +530,23 @@ func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, r
_ = binary.Write(&buffer, common.Endian, data)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, data)
case schemapb.DataType_Array:
data := &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{1, 2, 3},
},
},
}
bytes, _ := proto.Marshal(data)
binary.Write(&buffer, common.Endian, bytes)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, data)
case schemapb.DataType_JSON:
data := []byte(`{"key":"value"}`)
binary.Write(&buffer, common.Endian, data)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, data)
}
}
return ret, pk, row
@ -752,6 +797,49 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
for nrows := 0; nrows < numRows; nrows++ {
columns[idx] = append(columns[idx], data[nrows*bVecDim/8:(nrows+1)*bVecDim/8])
}
case schemapb.DataType_Array:
data := generateInt32ArrayList(numRows)
f := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: field.GetName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: data,
ElementType: schemapb.DataType_Int32,
},
},
},
},
FieldId: field.FieldID,
}
msg.FieldsData = append(msg.FieldsData, f)
for _, d := range data {
columns[idx] = append(columns[idx], d)
}
case schemapb.DataType_JSON:
data := generateBytesArray(numRows)
f := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: field.GetName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: data,
},
},
},
},
FieldId: field.FieldID,
}
msg.FieldsData = append(msg.FieldsData, f)
for _, d := range data {
columns[idx] = append(columns[idx], d)
}
}
}
@ -761,6 +849,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
func TestRowBasedInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim)
idata, err := RowBasedInsertMsgToInsertData(msg, schema)
@ -797,6 +886,7 @@ func TestColumnBasedInsertMsgToInsertData(t *testing.T) {
func TestInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim)
idata, err := InsertMsgToInsertData(msg, schema)
@ -804,7 +894,7 @@ func TestInsertMsgToInsertData(t *testing.T) {
for idx, fID := range fieldIDs {
column := columns[idx]
fData, ok := idata.Data[fID]
assert.True(t, ok)
assert.True(t, ok, "fID =", fID)
assert.Equal(t, len(column), fData.RowNum())
for j := range column {
assert.Equal(t, fData.GetRow(j), column[j])
@ -871,6 +961,20 @@ func TestMergeInsertData(t *testing.T) {
Data: []float32{0},
Dim: 1,
},
ArrayField: &ArrayFieldData{
Data: []*schemapb.ScalarField{
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{1, 2, 3},
},
},
},
},
},
JSONField: &JSONFieldData{
Data: [][]byte{[]byte(`{"key":"value"}`)},
},
},
Infos: nil,
}
@ -914,6 +1018,20 @@ func TestMergeInsertData(t *testing.T) {
Data: []float32{0},
Dim: 1,
},
ArrayField: &ArrayFieldData{
Data: []*schemapb.ScalarField{
{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{4, 5, 6},
},
},
},
},
},
JSONField: &JSONFieldData{
Data: [][]byte{[]byte(`{"hello":"world"}`)},
},
},
Infos: nil,
}
@ -967,6 +1085,15 @@ func TestMergeInsertData(t *testing.T) {
f, ok = merged.Data[FloatVectorField]
assert.True(t, ok)
assert.Equal(t, []float32{0, 0}, f.(*FloatVectorFieldData).Data)
f, ok = merged.Data[ArrayField]
assert.True(t, ok)
assert.Equal(t, []int32{1, 2, 3}, f.(*ArrayFieldData).Data[0].GetIntData().GetData())
assert.Equal(t, []int32{4, 5, 6}, f.(*ArrayFieldData).Data[1].GetIntData().GetData())
f, ok = merged.Data[JSONField]
assert.True(t, ok)
assert.EqualValues(t, [][]byte{[]byte(`{"key":"value"}`), []byte(`{"hello":"world"}`)}, f.(*JSONFieldData).Data)
}
func TestGetPkFromInsertData(t *testing.T) {

4
internal/util/funcutil/func.go
View File

@ -320,6 +320,10 @@ func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
fieldNumRows = getNumRowsOfScalarField(scalarField.GetDoubleData().Data)
case *schemapb.ScalarField_StringData:
fieldNumRows = getNumRowsOfScalarField(scalarField.GetStringData().Data)
case *schemapb.ScalarField_ArrayData:
fieldNumRows = getNumRowsOfScalarField(scalarField.GetArrayData().Data)
case *schemapb.ScalarField_JsonData:
fieldNumRows = getNumRowsOfScalarField(scalarField.GetJsonData().Data)
default:
return 0, fmt.Errorf("%s is not supported now", scalarType)
}

30
internal/util/typeutil/gen_empty_field_data.go
View File

@ -113,6 +113,32 @@ func genEmptyVarCharFieldData(field *schemapb.FieldSchema) *schemapb.FieldData {
}
}
func genEmptyArrayFieldData(field *schemapb.FieldSchema) *schemapb.FieldData {
return &schemapb.FieldData{
Type: field.GetDataType(),
FieldName: field.GetName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{ArrayData: &schemapb.ArrayArray{Data: nil}},
},
},
FieldId: field.GetFieldID(),
}
}
func genEmptyJSONFieldData(field *schemapb.FieldSchema) *schemapb.FieldData {
return &schemapb.FieldData{
Type: field.GetDataType(),
FieldName: field.GetName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{JsonData: &schemapb.JSONArray{Data: nil}},
},
},
FieldId: field.GetFieldID(),
}
}
func genEmptyBinaryVectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dim, err := GetDim(field)
if err != nil {
@ -168,6 +194,10 @@ func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error)
return genEmptyDoubleFieldData(field), nil
case schemapb.DataType_VarChar:
return genEmptyVarCharFieldData(field), nil
case schemapb.DataType_Array:
return genEmptyArrayFieldData(field), nil
case schemapb.DataType_JSON:
return genEmptyJSONFieldData(field), nil
case schemapb.DataType_BinaryVector:
return genEmptyBinaryVectorFieldData(field)
case schemapb.DataType_FloatVector:

2
internal/util/typeutil/result_helper_test.go
View File

@ -26,6 +26,8 @@ func TestGenEmptyFieldData(t *testing.T) {
schemapb.DataType_Float,
schemapb.DataType_Double,
schemapb.DataType_VarChar,
schemapb.DataType_Array,
schemapb.DataType_JSON,
}
allUnsupportedTypes := []schemapb.DataType{
schemapb.DataType_String,

80
internal/util/typeutil/schema.go
View File

@ -29,6 +29,8 @@ import (
"go.uber.org/zap"
)
const DynamicFieldMaxLength = 512
func GetAvgLengthOfVarLengthField(fieldSchema *schemapb.FieldSchema) (int, error) {
maxLength := 0
var err error
@ -38,10 +40,9 @@ func GetAvgLengthOfVarLengthField(fieldSchema *schemapb.FieldSchema) (int, error
paramsMap[p.Key] = p.Value
}
maxLengthPerRowKey := "max_length"
switch fieldSchema.DataType {
case schemapb.DataType_VarChar:
maxLengthPerRowKey := "max_length"
maxLengthPerRowValue, ok := paramsMap[maxLengthPerRowKey]
if !ok {
return 0, fmt.Errorf("the max_length was not specified, field type is %s", fieldSchema.DataType.String())
@ -50,6 +51,8 @@ func GetAvgLengthOfVarLengthField(fieldSchema *schemapb.FieldSchema) (int, error
if err != nil {
return 0, err
}
case schemapb.DataType_Array, schemapb.DataType_JSON:
return DynamicFieldMaxLength, nil
default:
return 0, fmt.Errorf("field %s is not a variable-length type", fieldSchema.DataType.String())
}
@ -75,7 +78,7 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
res += 4
case schemapb.DataType_Int64, schemapb.DataType_Double:
res += 8
case schemapb.DataType_VarChar:
case schemapb.DataType_VarChar, schemapb.DataType_Array, schemapb.DataType_JSON:
maxLengthPerRow, err := GetAvgLengthOfVarLengthField(fs)
if err != nil {
return 0, err
@ -108,6 +111,42 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
return res, nil
}
func CalcColumnSize(column *schemapb.FieldData) int {
res := 0
switch column.GetType() {
case schemapb.DataType_Bool:
res += len(column.GetScalars().GetBoolData().GetData())
case schemapb.DataType_Int8:
res += len(column.GetScalars().GetIntData().GetData())
case schemapb.DataType_Int16:
res += len(column.GetScalars().GetIntData().GetData()) * 2
case schemapb.DataType_Int32:
res += len(column.GetScalars().GetIntData().GetData()) * 4
case schemapb.DataType_Int64:
res += len(column.GetScalars().GetLongData().GetData()) * 8
case schemapb.DataType_Float:
res += len(column.GetScalars().GetFloatData().GetData()) * 4
case schemapb.DataType_Double:
res += len(column.GetScalars().GetDoubleData().GetData()) * 8
case schemapb.DataType_VarChar:
for _, str := range column.GetScalars().GetStringData().GetData() {
res += len(str)
}
case schemapb.DataType_Array:
for _, array := range column.GetScalars().GetArrayData().GetData() {
res += CalcColumnSize(&schemapb.FieldData{
Field: &schemapb.FieldData_Scalars{Scalars: array},
Type: column.GetScalars().GetArrayData().GetElementType(),
})
}
case schemapb.DataType_JSON:
for _, str := range column.GetScalars().GetJsonData().GetData() {
res += len(str)
}
}
return res
}
func EstimateEntitySize(fieldsData []*schemapb.FieldData, rowOffset int) (int, error) {
res := 0
for _, fs := range fieldsData {
@ -124,8 +163,21 @@ func EstimateEntitySize(fieldsData []*schemapb.FieldData, rowOffset int) (int, e
if rowOffset >= len(fs.GetScalars().GetStringData().GetData()) {
return 0, fmt.Errorf("offset out range of field datas")
}
//TODO:: check len(varChar) <= maxLengthPerRow
res += len(fs.GetScalars().GetStringData().Data[rowOffset])
case schemapb.DataType_Array:
if rowOffset >= len(fs.GetScalars().GetArrayData().GetData()) {
return 0, fmt.Errorf("offset out range of field datas")
}
array := fs.GetScalars().GetArrayData().GetData()[rowOffset]
res += CalcColumnSize(&schemapb.FieldData{
Field: &schemapb.FieldData_Scalars{Scalars: array},
Type: fs.GetScalars().GetArrayData().GetElementType(),
})
case schemapb.DataType_JSON:
if rowOffset >= len(fs.GetScalars().GetJsonData().GetData()) {
return 0, fmt.Errorf("offset out range of field datas")
}
res += len(fs.GetScalars().GetJsonData().GetData()[rowOffset])
case schemapb.DataType_BinaryVector:
res += int(fs.GetVectors().GetDim())
case schemapb.DataType_FloatVector:
@ -348,6 +400,26 @@ func AppendFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData, idx i
} else {
dstScalar.GetStringData().Data = append(dstScalar.GetStringData().Data, srcScalar.StringData.Data[idx])
}
case *schemapb.ScalarField_ArrayData:
if dstScalar.GetArrayData() == nil {
dstScalar.Data = &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: []*schemapb.ScalarField{srcScalar.ArrayData.Data[idx]},
},
}
} else {
dstScalar.GetArrayData().Data = append(dstScalar.GetArrayData().Data, srcScalar.ArrayData.Data[idx])
}
case *schemapb.ScalarField_JsonData:
if dstScalar.GetJsonData() == nil {
dstScalar.Data = &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: [][]byte{srcScalar.JsonData.Data[idx]},
},
}
} else {
dstScalar.GetJsonData().Data = append(dstScalar.GetJsonData().Data, srcScalar.JsonData.Data[idx])
}
default:
log.Error("Not supported field type", zap.String("field type", fieldData.Type.String()))
}

226
internal/util/typeutil/schema_test.go
View File

@ -17,6 +17,7 @@
package typeutil
import (
"encoding/binary"
"reflect"
"testing"
@ -124,12 +125,23 @@ func TestSchema(t *testing.T) {
},
},
},
{
FieldID: 109,
Name: "field_array",
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int32,
},
{
FieldID: 110,
Name: "field_json",
DataType: schemapb.DataType_JSON,
},
},
}
t.Run("EstimateSizePerRecord", func(t *testing.T) {
size, err := EstimateSizePerRecord(schema)
assert.Equal(t, 680, size)
assert.Equal(t, 680+DynamicFieldMaxLength*2, size)
assert.Nil(t, err)
})
@ -361,6 +373,44 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_Int8:
data := []int32{}
for _, v := range fieldValue.([]int8) {
data = append(data, int32(v))
}
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Int8,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: data,
},
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_Int16:
data := []int32{}
for _, v := range fieldValue.([]int16) {
data = append(data, int32(v))
}
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Int16,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: data,
},
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_Int32:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Int32,
@ -421,6 +471,21 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_VarChar:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_VarChar,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: fieldValue.([]string),
},
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_BinaryVector:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
@ -451,6 +516,49 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_Array:
data := fieldValue.([][]int32)
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: []*schemapb.ScalarField{},
ElementType: schemapb.DataType_Int32,
},
},
},
},
}
for _, list := range data {
arrayList := fieldData.GetScalars().GetArrayData()
arrayList.Data = append(arrayList.Data, &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: list,
},
},
})
}
case schemapb.DataType_JSON:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_JSON,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: fieldValue.([][]byte),
},
},
},
},
FieldId: fieldID,
}
default:
log.Error("not supported field type", zap.String("field type", fieldType.String()))
}
@ -787,3 +895,119 @@ func TestComparePk(t *testing.T) {
less = ComparePKInSlice(strPks, 2, 1)
assert.False(t, less)
}
func TestCalcColumnSize(t *testing.T) {
fieldValues := map[int64]any{
100: []int8{0, 1},
101: []int16{0, 1},
102: []int32{0, 1},
103: []int64{0, 1},
104: []float32{0, 1},
105: []float64{0, 1},
106: []string{"0", "1"},
107: []float32{0, 1, 2, 3},
109: [][]int32{{1, 2, 3}, {4, 5, 6}},
110: [][]byte{[]byte(`{"key":"value"}`), []byte(`{"hello":"world"}`)},
}
schema := &schemapb.CollectionSchema{
Name: "testColl",
Description: "",
AutoID: false,
Fields: []*schemapb.FieldSchema{
{
FieldID: 100,
Name: "field_int8",
IsPrimaryKey: false,
DataType: schemapb.DataType_Int8,
},
{
FieldID: 101,
Name: "field_int16",
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Int16,
},
{
FieldID: 102,
Name: "field_int32",
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Int32,
},
{
FieldID: 103,
Name: "field_int64",
IsPrimaryKey: true,
Description: "",
DataType: schemapb.DataType_Int64,
},
{
FieldID: 104,
Name: "field_float",
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Float,
},
{
FieldID: 105,
Name: "field_double",
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Double,
},
{
FieldID: 106,
Name: "field_string",
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_VarChar,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "max_length",
Value: "125",
},
},
},
{
FieldID: 109,
Name: "field_array",
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int32,
},
{
FieldID: 110,
Name: "field_json",
DataType: schemapb.DataType_JSON,
},
},
}
for _, field := range schema.GetFields() {
values := fieldValues[field.GetFieldID()]
fieldData := genFieldData(field.GetName(), field.GetFieldID(), field.GetDataType(), values, 0)
size := CalcColumnSize(fieldData)
expected := 0
switch field.GetDataType() {
case schemapb.DataType_VarChar:
data := values.([]string)
for _, v := range data {
expected += len(v)
}
case schemapb.DataType_Array:
data := values.([][]int32)
for _, v := range data {
expected += binary.Size(v)
}
case schemapb.DataType_JSON:
data := values.([][]byte)
for _, v := range data {
expected += len(v)
}
default:
expected = binary.Size(fieldValues[field.GetFieldID()])
}
assert.Equal(t, expected, size, field.GetName())
}
}

12
scripts/run_go_codecov.sh
View File

@ -37,6 +37,18 @@ for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/g
rm profile.out
fi
done
pushd pkg
for d in $(go list ./... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do
go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=../profile.out -covermode=atomic "$d"
if [ -f ../profile.out ]; then
grep -v kafka ../profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ../${FILE_COVERAGE_INFO}
rm ../profile.out
fi
done
popd
endTime=`date +%s`
echo "Total time for go unittest:" $(($endTime-$beginTime)) "s"
# generate html report
go tool cover -html=./${FILE_COVERAGE_INFO} -o ./${FILE_COVERAGE_HTML}