milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename variable names for better readibility (#15700) 

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
pull/15706/head
Cai Yudong 2022-02-22 22:15:52 +08:00 committed by GitHub
parent 1bb5ef0236
commit 54b8b24151
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 146 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/core/src/common/Types.h
View File

@ -22,6 +22,7 @@
#include <utility>
#include <vector>
#include <boost/align/aligned_allocator.hpp>
#include <boost/dynamic_bitset.hpp>
#include <NamedType/named_type.hpp>
#include "knowhere/utils/BitsetView.h"
@ -89,6 +90,7 @@ struct SearchResult {
};
using SearchResultPtr = std::shared_ptr<SearchResult>;
using SearchResultOpt = std::optional<SearchResult>;
struct RetrieveResult {
RetrieveResult() = default;
@ -100,6 +102,7 @@ struct RetrieveResult {
};
using RetrieveResultPtr = std::shared_ptr<RetrieveResult>;
using RetrieveResultOpt = std::optional<RetrieveResult>;
namespace impl {
// hide identifier name to make auto-completion happy
@ -124,4 +127,7 @@ BitsetSubView(const BitsetView& view, int64_t offset, int64_t size) {
return BitsetView(view.data() + offset / 8, size);
}
using BitsetType = boost::dynamic_bitset<>;
using BitsetTypeOpt = std::optional<BitsetType>;
} // namespace milvus

3
internal/core/src/query/PlanImpl.h
View File

@ -16,7 +16,6 @@
#include <optional>
#include <string>
#include <vector>
#include <boost/dynamic_bitset.hpp>
#include "Plan.h"
#include "PlanNode.h"
@ -39,7 +38,7 @@ struct ExtractedPlanInfo {
}
public:
boost::dynamic_bitset<> involved_fields_;
BitsetType involved_fields_;
};
struct Plan {

5
internal/core/src/query/PlanProto.h
View File

@ -26,8 +26,8 @@ class ProtoParser {
explicit ProtoParser(const Schema& schema) : schema(schema) {
}
// ExprPtr
// ExprFromProto(const proto::plan::Expr& expr_proto);
// ExprPtr
// ExprFromProto(const proto::plan::Expr& expr_proto);
ExprPtr
ParseUnaryRangeExpr(const proto::plan::UnaryRangeExpr& expr_pb);
@ -64,7 +64,6 @@ class ProtoParser {
private:
const Schema& schema;
// boost::dynamic_bitset<> involved_fields;
};
} // namespace milvus::query

3
internal/core/src/query/SearchOnSealed.h
View File

@ -20,8 +20,7 @@
namespace milvus::query {
using BitsetChunk = boost::dynamic_bitset<>;
using BitsetSimple = std::deque<BitsetChunk>;
using BitsetSimple = std::deque<BitsetType>;
aligned_vector<uint8_t>
AssembleNegBitset(const BitsetSimple& bitmap_simple);

28
internal/core/src/query/generated/ExecExprVisitor.h
View File

@ -13,7 +13,6 @@
// Generated File
// DO NOT EDIT
#include <optional>
#include <boost/dynamic_bitset.hpp>
#include <boost/variant.hpp>
#include <utility>
#include <deque>
@ -43,45 +42,46 @@ class ExecExprVisitor : public ExprVisitor {
visit(CompareExpr& expr) override;
public:
using RetType = boost::dynamic_bitset<>;
ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count, Timestamp timestamp)
: segment_(segment), row_count_(row_count), timestamp_(timestamp) {
}
RetType
BitsetType
call_child(Expr& expr) {
Assert(!ret_.has_value());
Assert(!bitset_opt_.has_value());
expr.accept(*this);
Assert(ret_.has_value());
auto res = std::move(ret_);
ret_ = std::nullopt;
Assert(bitset_opt_.has_value());
auto res = std::move(bitset_opt_);
bitset_opt_ = std::nullopt;
return std::move(res.value());
}
public:
template <typename T, typename IndexFunc, typename ElementFunc>
auto
ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc func, ElementFunc element_func) -> RetType;
ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc func, ElementFunc element_func) -> BitsetType;
template <typename T>
auto
ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> RetType;
ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> RetType;
ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType;
ExecTermVisitorImpl(TermExpr& expr_raw) -> BitsetType;
template <typename CmpFunc>
auto
ExecCompareExprDispatcher(CompareExpr& expr, CmpFunc cmp_func) -> RetType;
ExecCompareExprDispatcher(CompareExpr& expr, CmpFunc cmp_func) -> BitsetType;
private:
const segcore::SegmentInternalInterface& segment_;
int64_t row_count_;
std::optional<RetType> ret_;
Timestamp timestamp_;
int64_t row_count_;
BitsetTypeOpt bitset_opt_;
};
} // namespace milvus::query

32
internal/core/src/query/generated/ExecPlanNodeVisitor.h
View File

@ -31,8 +31,6 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
visit(RetrievePlanNode& node) override;
public:
using RetType = SearchResult;
using RetrieveRetType = RetrieveResult;
ExecPlanNodeVisitor(const segcore::SegmentInterface& segment,
Timestamp timestamp,
const PlaceholderGroup& placeholder_group)
@ -43,26 +41,27 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
: segment_(segment), timestamp_(timestamp) {
}
RetType
SearchResult
get_moved_result(PlanNode& node) {
assert(!ret_.has_value());
assert(!search_result_opt_.has_value());
node.accept(*this);
assert(ret_.has_value());
auto ret = std::move(ret_).value();
ret_ = std::nullopt;
assert(search_result_opt_.has_value());
auto ret = std::move(search_result_opt_).value();
search_result_opt_.reset();
search_result_opt_ = std::nullopt;
return ret;
}
RetrieveRetType
RetrieveResult
get_retrieve_result(PlanNode& node) {
assert(!retrieve_ret_.has_value());
assert(!retrieve_result_opt_.has_value());
std::cout.flush();
node.accept(*this);
assert(retrieve_ret_.has_value());
auto retrieve_ret = std::move(retrieve_ret_).value();
retrieve_ret_.reset();
retrieve_ret_ = std::nullopt;
return retrieve_ret;
assert(retrieve_result_opt_.has_value());
auto ret = std::move(retrieve_result_opt_).value();
retrieve_result_opt_.reset();
retrieve_result_opt_ = std::nullopt;
return ret;
}
private:
@ -71,12 +70,11 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
VectorVisitorImpl(VectorPlanNode& node);
private:
// std::optional<RetType> ret_;
const segcore::SegmentInterface& segment_;
Timestamp timestamp_;
PlaceholderGroup placeholder_group_;
std::optional<RetType> ret_;
std::optional<RetrieveResult> retrieve_ret_;
SearchResultOpt search_result_opt_;
RetrieveResultOpt retrieve_result_opt_;
};
} // namespace milvus::query

15
internal/core/src/query/generated/ShowExprVisitor.h
View File

@ -38,16 +38,13 @@ class ShowExprVisitor : public ExprVisitor {
visit(CompareExpr& expr) override;
public:
using RetType = Json;
public:
RetType
Json
call_child(Expr& expr) {
assert(!ret_.has_value());
assert(!json_opt_.has_value());
expr.accept(*this);
assert(ret_.has_value());
auto ret = std::move(ret_);
ret_ = std::nullopt;
assert(json_opt_.has_value());
auto ret = std::move(json_opt_);
json_opt_ = std::nullopt;
return std::move(ret.value());
}
@ -67,6 +64,6 @@ class ShowExprVisitor : public ExprVisitor {
}
private:
std::optional<RetType> ret_;
std::optional<json> json_opt_;
};
} // namespace milvus::query

1
internal/core/src/query/generated/VerifyExprVisitor.h
View File

@ -13,7 +13,6 @@
// Generated File
// DO NOT EDIT
#include <optional>
#include <boost/dynamic_bitset.hpp>
#include <boost/variant.hpp>
#include <utility>
#include <deque>

75
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -13,7 +13,6 @@
#include <optional>
#include <unordered_set>
#include <utility>
#include <boost/dynamic_bitset.hpp>
#include <boost/variant.hpp>
#include "query/ExprImpl.h"
@ -26,46 +25,46 @@ namespace milvus::query {
namespace impl {
class ExecExprVisitor : ExprVisitor {
public:
using RetType = boost::dynamic_bitset<>;
ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count, Timestamp timestamp)
: segment_(segment), row_count_(row_count), timestamp_(timestamp) {
}
RetType
BitsetType
call_child(Expr& expr) {
AssertInfo(!ret_.has_value(), "[ExecExprVisitor]Bitset already has value before accept");
AssertInfo(!bitset_opt_.has_value(), "[ExecExprVisitor]Bitset already has value before accept");
expr.accept(*this);
AssertInfo(ret_.has_value(), "[ExecExprVisitor]Bitset doesn't have value after accept");
auto res = std::move(ret_);
ret_ = std::nullopt;
AssertInfo(bitset_opt_.has_value(), "[ExecExprVisitor]Bitset doesn't have value after accept");
auto res = std::move(bitset_opt_);
bitset_opt_ = std::nullopt;
return std::move(res.value());
}
public:
template <typename T, typename IndexFunc, typename ElementFunc>
auto
ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc func, ElementFunc element_func) -> RetType;
ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc func, ElementFunc element_func) -> BitsetType;
template <typename T>
auto
ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> RetType;
ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> RetType;
ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> BitsetType;
template <typename T>
auto
ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType;
ExecTermVisitorImpl(TermExpr& expr_raw) -> BitsetType;
template <typename CmpFunc>
auto
ExecCompareExprDispatcher(CompareExpr& expr, CmpFunc cmp_func) -> RetType;
ExecCompareExprDispatcher(CompareExpr& expr, CmpFunc cmp_func) -> BitsetType;
private:
const segcore::SegmentInternalInterface& segment_;
int64_t row_count_;
std::optional<RetType> ret_;
Timestamp timestamp_;
BitsetTypeOpt bitset_opt_;
};
} // namespace impl
@ -73,7 +72,7 @@ void
ExecExprVisitor::visit(LogicalUnaryExpr& expr) {
using OpType = LogicalUnaryExpr::OpType;
auto child_res = call_child(*expr.child_);
RetType res = std::move(child_res);
BitsetType res = std::move(child_res);
switch (expr.op_type_) {
case OpType::LogicalNot: {
res.flip();
@ -84,7 +83,7 @@ ExecExprVisitor::visit(LogicalUnaryExpr& expr) {
}
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
void
@ -116,12 +115,12 @@ ExecExprVisitor::visit(LogicalBinaryExpr& expr) {
}
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
static auto
Assemble(const std::deque<boost::dynamic_bitset<>>& srcs) -> boost::dynamic_bitset<> {
boost::dynamic_bitset<> res;
Assemble(const std::deque<BitsetType>& srcs) -> BitsetType {
BitsetType res;
int64_t total_size = 0;
for (auto& chunk : srcs) {
@ -142,13 +141,13 @@ Assemble(const std::deque<boost::dynamic_bitset<>>& srcs) -> boost::dynamic_bits
template <typename T, typename IndexFunc, typename ElementFunc>
auto
ExecExprVisitor::ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc index_func, ElementFunc element_func)
-> RetType {
-> BitsetType {
auto& schema = segment_.get_schema();
auto& field_meta = schema[field_offset];
auto indexing_barrier = segment_.num_chunk_index(field_offset);
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
std::deque<boost::dynamic_bitset<>> results;
std::deque<BitsetType> results;
using Index = knowhere::scalar::StructuredIndex<T>;
for (auto chunk_id = 0; chunk_id < indexing_barrier; ++chunk_id) {
@ -161,7 +160,7 @@ ExecExprVisitor::ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc index_
}
for (auto chunk_id = indexing_barrier; chunk_id < num_chunk; ++chunk_id) {
auto this_size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
boost::dynamic_bitset<> result(this_size);
BitsetType result(this_size);
auto chunk = segment_.chunk_data<T>(field_offset, chunk_id);
const T* data = chunk.data();
for (int index = 0; index < this_size; ++index) {
@ -179,7 +178,7 @@ ExecExprVisitor::ExecRangeVisitorImpl(FieldOffset field_offset, IndexFunc index_
#pragma ide diagnostic ignored "Simplify"
template <typename T>
auto
ExecExprVisitor::ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> RetType {
ExecExprVisitor::ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<UnaryRangeExprImpl<T>&>(expr_raw);
using Index = knowhere::scalar::StructuredIndex<T>;
using Operator = knowhere::scalar::OperatorType;
@ -227,7 +226,7 @@ ExecExprVisitor::ExecUnaryRangeVisitorDispatcher(UnaryRangeExpr& expr_raw) -> Re
#pragma ide diagnostic ignored "Simplify"
template <typename T>
auto
ExecExprVisitor::ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> RetType {
ExecExprVisitor::ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<BinaryRangeExprImpl<T>&>(expr_raw);
using Index = knowhere::scalar::StructuredIndex<T>;
using Operator = knowhere::scalar::OperatorType;
@ -237,7 +236,7 @@ ExecExprVisitor::ExecBinaryRangeVisitorDispatcher(BinaryRangeExpr& expr_raw) ->
T val2 = expr.upper_value_;
// TODO: disable check?
if (val1 > val2 || (val1 == val2 && !(lower_inclusive && upper_inclusive))) {
RetType res(row_count_, false);
BitsetType res(row_count_, false);
return res;
}
auto index_func = [=](Index* index) { return index->Range(val1, lower_inclusive, val2, upper_inclusive); };
@ -262,7 +261,7 @@ ExecExprVisitor::visit(UnaryRangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_offset_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type");
RetType res;
BitsetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
res = ExecUnaryRangeVisitorDispatcher<bool>(expr);
@ -296,7 +295,7 @@ ExecExprVisitor::visit(UnaryRangeExpr& expr) {
PanicInfo("unsupported");
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
void
@ -304,7 +303,7 @@ ExecExprVisitor::visit(BinaryRangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_offset_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type");
RetType res;
BitsetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
res = ExecBinaryRangeVisitorDispatcher<bool>(expr);
@ -338,7 +337,7 @@ ExecExprVisitor::visit(BinaryRangeExpr& expr) {
PanicInfo("unsupported");
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
template <typename Op>
@ -357,11 +356,11 @@ struct relational {
template <typename Op>
auto
ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> RetType {
ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> BitsetType {
using number = boost::variant<bool, int8_t, int16_t, int32_t, int64_t, float, double>;
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
std::deque<RetType> bitsets;
std::deque<BitsetType> bitsets;
for (int64_t chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
auto getChunkData = [&, chunk_id](DataType type, FieldOffset offset) -> std::function<const number(int)> {
@ -401,7 +400,7 @@ ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> RetType
auto left = getChunkData(expr.left_data_type_, expr.left_field_offset_);
auto right = getChunkData(expr.right_data_type_, expr.right_field_offset_);
boost::dynamic_bitset<> bitset(size);
BitsetType bitset(size);
for (int i = 0; i < size; ++i) {
bool is_in = boost::apply_visitor(relational<decltype(op)>{}, left(i), right(i));
bitset[i] = is_in;
@ -423,7 +422,7 @@ ExecExprVisitor::visit(CompareExpr& expr) {
AssertInfo(expr.right_data_type_ == right_field_meta.get_data_type(),
"[ExecExprVisitor]right data type not equal to right field mata type");
RetType res;
BitsetType res;
switch (expr.op_type_) {
case OpType::Equal: {
res = ExecCompareExprDispatcher(expr, std::equal_to<>{});
@ -454,12 +453,12 @@ ExecExprVisitor::visit(CompareExpr& expr) {
}
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
template <typename T>
auto
ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType {
ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> BitsetType {
auto& expr = static_cast<TermExprImpl<T>&>(expr_raw);
auto& schema = segment_.get_schema();
@ -467,13 +466,13 @@ ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType {
auto& field_meta = schema[field_offset];
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
std::deque<RetType> bitsets;
std::deque<BitsetType> bitsets;
std::unordered_set<T> term_set(expr.terms_.begin(), expr.terms_.end());
for (int64_t chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
Span<T> chunk = segment_.chunk_data<T>(field_offset, chunk_id);
auto chunk_data = chunk.data();
auto size = (chunk_id == num_chunk - 1) ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
boost::dynamic_bitset<> bitset(size);
BitsetType bitset(size);
for (int i = 0; i < size; ++i) {
bitset[i] = (term_set.find(chunk_data[i]) != term_set.end());
}
@ -489,7 +488,7 @@ ExecExprVisitor::visit(TermExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_offset_];
AssertInfo(expr.data_type_ == field_meta.get_data_type(),
"[ExecExprVisitor]DataType of expr isn't field_meta data type ");
RetType res;
BitsetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
res = ExecTermVisitorImpl<bool>(expr);
@ -523,6 +522,6 @@ ExecExprVisitor::visit(TermExpr& expr) {
PanicInfo("unsupported");
}
AssertInfo(res.size() == row_count_, "[ExecExprVisitor]Size of results not equal row count");
ret_ = std::move(res);
bitset_opt_ = std::move(res);
}
} // namespace milvus::query

49
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -26,21 +26,19 @@ namespace impl {
// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
class ExecPlanNodeVisitor : PlanNodeVisitor {
public:
using RetType = SearchResult;
ExecPlanNodeVisitor(const segcore::SegmentInterface& segment,
Timestamp timestamp,
const PlaceholderGroup& placeholder_group)
: segment_(segment), timestamp_(timestamp), placeholder_group_(placeholder_group) {
}
// using RetType = nlohmann::json;
RetType
SearchResult
get_moved_result(PlanNode& node) {
assert(!ret_.has_value());
assert(!search_result_opt_.has_value());
node.accept(*this);
assert(ret_.has_value());
auto ret = std::move(ret_).value();
ret_ = std::nullopt;
assert(search_result_opt_.has_value());
auto ret = std::move(search_result_opt_).value();
search_result_opt_ = std::nullopt;
return ret;
}
@ -50,12 +48,11 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
VectorVisitorImpl(VectorPlanNode& node);
private:
// std::optional<RetType> ret_;
const segcore::SegmentInterface& segment_;
Timestamp timestamp_;
const PlaceholderGroup& placeholder_group_;
std::optional<RetType> ret_;
SearchResultOpt search_result_opt_;
};
} // namespace impl
@ -74,15 +71,15 @@ template <typename VectorType>
void
ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
// TODO: optimize here, remove the dynamic cast
assert(!ret_.has_value());
assert(!search_result_opt_.has_value());
auto segment = dynamic_cast<const segcore::SegmentInternalInterface*>(&segment_);
AssertInfo(segment, "support SegmentSmallIndex Only");
RetType ret;
SearchResult search_result;
auto& ph = placeholder_group_.at(0);
auto src_data = ph.get_blob<EmbeddedType<VectorType>>();
auto num_queries = ph.num_of_queries_;
boost::dynamic_bitset<> bitset_holder;
BitsetType bitset_holder;
BitsetView view;
// TODO: add API to unify row_count
// auto row_count = segment->get_row_count();
@ -90,14 +87,13 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
// skip all calculation
if (active_count == 0) {
ret_ = empty_search_result(num_queries, node.search_info_.topk_, node.search_info_.round_decimal_,
node.search_info_.metric_type_);
search_result_opt_ = empty_search_result(num_queries, node.search_info_.topk_, node.search_info_.round_decimal_,
node.search_info_.metric_type_);
return;
}
if (node.predicate_.has_value()) {
ExecExprVisitor::RetType expr_ret =
ExecExprVisitor(*segment, active_count, timestamp_).call_child(*node.predicate_.value());
BitsetType expr_ret = ExecExprVisitor(*segment, active_count, timestamp_).call_child(*node.predicate_.value());
bitset_holder = std::move(expr_ret);
} else {
bitset_holder.resize(active_count, true);
@ -111,29 +107,29 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
auto final_bitset = segment->get_filtered_bitmap(view, active_count, timestamp_);
segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, final_bitset, ret);
segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, final_bitset,
search_result);
ret_ = ret;
search_result_opt_ = std::move(search_result);
}
void
ExecPlanNodeVisitor::visit(RetrievePlanNode& node) {
assert(!retrieve_ret_.has_value());
assert(!retrieve_result_opt_.has_value());
auto segment = dynamic_cast<const segcore::SegmentInternalInterface*>(&segment_);
AssertInfo(segment, "Support SegmentSmallIndex Only");
RetrieveRetType ret;
RetrieveResult retrieve_result;
boost::dynamic_bitset<> bitset_holder;
BitsetType bitset_holder;
auto active_count = segment->get_active_count(timestamp_);
if (active_count == 0) {
retrieve_ret_ = ret;
retrieve_result_opt_ = std::move(retrieve_result);
return;
}
if (node.predicate_ != nullptr) {
ExecExprVisitor::RetType expr_ret =
ExecExprVisitor(*segment, active_count, timestamp_).call_child(*(node.predicate_));
BitsetType expr_ret = ExecExprVisitor(*segment, active_count, timestamp_).call_child(*(node.predicate_));
bitset_holder = std::move(expr_ret);
}
@ -148,8 +144,9 @@ ExecPlanNodeVisitor::visit(RetrievePlanNode& node) {
auto final_bitset = segment->get_filtered_bitmap(view, active_count, timestamp_);
auto seg_offsets = std::move(segment->search_ids(final_bitset, MAX_TIMESTAMP));
ret.result_offsets_.assign((int64_t*)seg_offsets.data(), (int64_t*)seg_offsets.data() + seg_offsets.size());
retrieve_ret_ = ret;
retrieve_result.result_offsets_.assign((int64_t*)seg_offsets.data(),
(int64_t*)seg_offsets.data() + seg_offsets.size());
retrieve_result_opt_ = std::move(retrieve_result);
}
void

48
internal/core/src/query/visitors/ShowExprVisitor.cpp
View File

@ -58,7 +58,7 @@ class ShowExprNodeVisitor : ExprVisitor {
void
ShowExprVisitor::visit(LogicalUnaryExpr& expr) {
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
using OpType = LogicalUnaryExpr::OpType;
// TODO: use magic_enum if available
@ -69,12 +69,12 @@ ShowExprVisitor::visit(LogicalUnaryExpr& expr) {
{"expr_type", "BoolUnary"},
{"op", op_name},
};
ret_ = this->combine(std::move(extra), expr);
json_opt_ = this->combine(std::move(extra), expr);
}
void
ShowExprVisitor::visit(LogicalBinaryExpr& expr) {
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
using OpType = LogicalBinaryExpr::OpType;
// TODO: use magic_enum if available
@ -95,7 +95,7 @@ ShowExprVisitor::visit(LogicalBinaryExpr& expr) {
{"expr_type", "BoolBinary"},
{"op", op_name},
};
ret_ = this->combine(std::move(extra), expr);
json_opt_ = this->combine(std::move(extra), expr);
}
template <typename T>
@ -108,7 +108,7 @@ TermExtract(const TermExpr& expr_raw) {
void
ShowExprVisitor::visit(TermExpr& expr) {
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(datatype_is_vector(expr.data_type_) == false, "[ShowExprVisitor]Data type of expr isn't vector type");
auto terms = [&] {
switch (expr.data_type_) {
@ -136,7 +136,7 @@ ShowExprVisitor::visit(TermExpr& expr) {
{"data_type", datatype_name(expr.data_type_)},
{"terms", std::move(terms)}};
ret_ = res;
json_opt_ = res;
}
template <typename T>
@ -156,29 +156,29 @@ UnaryRangeExtract(const UnaryRangeExpr& expr_raw) {
void
ShowExprVisitor::visit(UnaryRangeExpr& expr) {
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(datatype_is_vector(expr.data_type_) == false, "[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.data_type_) {
case DataType::BOOL:
ret_ = UnaryRangeExtract<bool>(expr);
json_opt_ = UnaryRangeExtract<bool>(expr);
return;
case DataType::INT8:
ret_ = UnaryRangeExtract<int8_t>(expr);
json_opt_ = UnaryRangeExtract<int8_t>(expr);
return;
case DataType::INT16:
ret_ = UnaryRangeExtract<int16_t>(expr);
json_opt_ = UnaryRangeExtract<int16_t>(expr);
return;
case DataType::INT32:
ret_ = UnaryRangeExtract<int32_t>(expr);
json_opt_ = UnaryRangeExtract<int32_t>(expr);
return;
case DataType::INT64:
ret_ = UnaryRangeExtract<int64_t>(expr);
json_opt_ = UnaryRangeExtract<int64_t>(expr);
return;
case DataType::DOUBLE:
ret_ = UnaryRangeExtract<double>(expr);
json_opt_ = UnaryRangeExtract<double>(expr);
return;
case DataType::FLOAT:
ret_ = UnaryRangeExtract<float>(expr);
json_opt_ = UnaryRangeExtract<float>(expr);
return;
default:
PanicInfo("unsupported type");
@ -204,29 +204,29 @@ BinaryRangeExtract(const BinaryRangeExpr& expr_raw) {
void
ShowExprVisitor::visit(BinaryRangeExpr& expr) {
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(datatype_is_vector(expr.data_type_) == false, "[ShowExprVisitor]Data type of expr isn't vector type");
switch (expr.data_type_) {
case DataType::BOOL:
ret_ = BinaryRangeExtract<bool>(expr);
json_opt_ = BinaryRangeExtract<bool>(expr);
return;
case DataType::INT8:
ret_ = BinaryRangeExtract<int8_t>(expr);
json_opt_ = BinaryRangeExtract<int8_t>(expr);
return;
case DataType::INT16:
ret_ = BinaryRangeExtract<int16_t>(expr);
json_opt_ = BinaryRangeExtract<int16_t>(expr);
return;
case DataType::INT32:
ret_ = BinaryRangeExtract<int32_t>(expr);
json_opt_ = BinaryRangeExtract<int32_t>(expr);
return;
case DataType::INT64:
ret_ = BinaryRangeExtract<int64_t>(expr);
json_opt_ = BinaryRangeExtract<int64_t>(expr);
return;
case DataType::DOUBLE:
ret_ = BinaryRangeExtract<double>(expr);
json_opt_ = BinaryRangeExtract<double>(expr);
return;
case DataType::FLOAT:
ret_ = BinaryRangeExtract<float>(expr);
json_opt_ = BinaryRangeExtract<float>(expr);
return;
default:
PanicInfo("unsupported type");
@ -237,7 +237,7 @@ void
ShowExprVisitor::visit(CompareExpr& expr) {
using proto::plan::OpType;
using proto::plan::OpType_Name;
AssertInfo(!ret_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
AssertInfo(!json_opt_.has_value(), "[ShowExprVisitor]Ret json already has value before visit");
Json res{{"expr_type", "Compare"},
{"left_field_offset", expr.left_field_offset_.get()},
@ -245,7 +245,7 @@ ShowExprVisitor::visit(CompareExpr& expr) {
{"right_field_offset", expr.right_field_offset_.get()},
{"right_data_type", datatype_name(expr.right_data_type_)},
{"op", OpType_Name(static_cast<OpType>(expr.op_type_))}};
ret_ = res;
json_opt_ = res;
}
} // namespace milvus::query

4
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -452,7 +452,7 @@ SegmentGrowingImpl::Insert(int64_t reserved_offset,
}
std::vector<SegOffset>
SegmentGrowingImpl::search_ids(const boost::dynamic_bitset<>& bitset, Timestamp timestamp) const {
SegmentGrowingImpl::search_ids(const BitsetType& bitset, Timestamp timestamp) const {
std::vector<SegOffset> res_offsets;
for (int i = 0; i < bitset.size(); i++) {
@ -534,7 +534,7 @@ SegmentGrowingImpl::get_active_count(Timestamp ts) const {
}
void
SegmentGrowingImpl::mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const {
SegmentGrowingImpl::mask_with_timestamps(BitsetType& bitset_chunk, Timestamp timestamp) const {
// DO NOTHING
}

4
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -165,7 +165,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
}
void
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const override;
mask_with_timestamps(BitsetType& bitset_chunk, Timestamp timestamp) const override;
void
vector_search(int64_t vec_count,
@ -190,7 +190,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
search_ids(const IdArray& id_array, Timestamp timestamp) const override;
std::vector<SegOffset>
search_ids(const boost::dynamic_bitset<>& view, Timestamp timestamp) const override;
search_ids(const BitsetType& view, Timestamp timestamp) const override;
std::vector<SegOffset>
search_ids(const BitsetView& view, Timestamp timestamp) const override;

4
internal/core/src/segcore/SegmentInterface.h
View File

@ -119,7 +119,7 @@ class SegmentInternalInterface : public SegmentInterface {
num_chunk_index(FieldOffset field_offset) const = 0;
virtual void
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const = 0;
mask_with_timestamps(BitsetType& bitset_chunk, Timestamp timestamp) const = 0;
// count of chunks
virtual int64_t
@ -133,7 +133,7 @@ class SegmentInternalInterface : public SegmentInterface {
get_active_count(Timestamp ts) const = 0;
virtual std::vector<SegOffset>
search_ids(const boost::dynamic_bitset<>& view, Timestamp timestamp) const = 0;
search_ids(const BitsetType& view, Timestamp timestamp) const = 0;
virtual std::vector<SegOffset>
search_ids(const BitsetView& view, Timestamp timestamp) const = 0;

8
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -18,12 +18,12 @@
namespace milvus::segcore {
static inline void
set_bit(boost::dynamic_bitset<>& bitset, FieldOffset field_offset, bool flag = true) {
set_bit(BitsetType& bitset, FieldOffset field_offset, bool flag = true) {
bitset[field_offset.get()] = flag;
}
static inline bool
get_bit(const boost::dynamic_bitset<>& bitset, FieldOffset field_offset) {
get_bit(const BitsetType& bitset, FieldOffset field_offset) {
return bitset[field_offset.get()];
}
@ -572,7 +572,7 @@ SegmentSealedImpl::Delete(int64_t reserved_offset,
}
std::vector<SegOffset>
SegmentSealedImpl::search_ids(const boost::dynamic_bitset<>& bitset, Timestamp timestamp) const {
SegmentSealedImpl::search_ids(const BitsetType& bitset, Timestamp timestamp) const {
std::vector<SegOffset> dst_offset;
for (int i = 0; i < bitset.size(); i++) {
if (bitset[i]) {
@ -618,7 +618,7 @@ SegmentSealedImpl::get_active_count(Timestamp ts) const {
return this->get_row_count();
}
void
SegmentSealedImpl::mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const {
SegmentSealedImpl::mask_with_timestamps(BitsetType& bitset_chunk, Timestamp timestamp) const {
// TODO change the
AssertInfo(this->timestamps_.size() == get_row_count(), "Timestamp size not equal to row count");
auto range = timestamp_index_.get_active_range(timestamp);

8
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -129,7 +129,7 @@ class SegmentSealedImpl : public SegmentSealed {
}
void
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const override;
mask_with_timestamps(BitsetType& bitset_chunk, Timestamp timestamp) const override;
void
vector_search(int64_t vec_count,
@ -160,15 +160,15 @@ class SegmentSealedImpl : public SegmentSealed {
search_ids(const BitsetView& view, Timestamp timestamp) const override;
std::vector<SegOffset>
search_ids(const boost::dynamic_bitset<>& view, Timestamp timestamp) const override;
search_ids(const BitsetType& view, Timestamp timestamp) const override;
// virtual void
// build_index_if_primary_key(FieldId field_id);
private:
// segment loading state
boost::dynamic_bitset<> field_data_ready_bitset_;
boost::dynamic_bitset<> vecindex_ready_bitset_;
BitsetType field_data_ready_bitset_;
BitsetType vecindex_ready_bitset_;
std::atomic<int> system_ready_count_ = 0;
// segment datas

4
internal/core/src/segcore/TimestampIndex.cpp
View File

@ -63,14 +63,14 @@ TimestampIndex::get_active_range(Timestamp query_timestamp) const {
return {start_locs_[block_id], start_locs_[block_id + 1]};
}
boost::dynamic_bitset<>
BitsetType
TimestampIndex::GenerateBitset(Timestamp query_timestamp,
std::pair<int64_t, int64_t> active_range,
const Timestamp* timestamps,
int64_t size) {
auto [beg, end] = active_range;
Assert(beg < end);
boost::dynamic_bitset<> bitset;
BitsetType bitset;
bitset.reserve(size);
bitset.resize(beg, true);
bitset.resize(size, false);

2
internal/core/src/segcore/TimestampIndex.h
View File

@ -34,7 +34,7 @@ class TimestampIndex {
std::pair<int64_t, int64_t>
get_active_range(Timestamp query_timestamp) const;
static boost::dynamic_bitset<>
static BitsetType
GenerateBitset(Timestamp query_timestamp,
std::pair<int64_t, int64_t> active_range,
const Timestamp* timestamps,