milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor expr executor to use single bitset (#6667) 

Signed-off-by: fluorinedog <fluorinedog@gmail.com>
pull/6779/head
FluorineDog 2021-07-23 15:30:12 +08:00 committed by GitHub
parent 5adb69161f
commit aba21baf82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 159 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/query/generated/ExecExprVisitor.h
View File

@ -14,6 +14,7 @@
// DO NOT EDIT
#include <optional>
#include <boost/dynamic_bitset.hpp>
#include <boost/variant.hpp>
#include <utility>
#include <deque>
#include "segcore/SegmentGrowingImpl.h"
@ -39,7 +40,7 @@ class ExecExprVisitor : public ExprVisitor {
visit(CompareExpr& expr) override;
public:
using RetType = std::deque<boost::dynamic_bitset<>>;
using RetType = boost::dynamic_bitset<>;
ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count, Timestamp timestamp)
: segment_(segment), row_count_(row_count), timestamp_(timestamp) {
}

1
internal/core/src/query/generated/VerifyExprVisitor.h
View File

@ -14,6 +14,7 @@
// DO NOT EDIT
#include <optional>
#include <boost/dynamic_bitset.hpp>
#include <boost/variant.hpp>
#include <utility>
#include <deque>
#include "segcore/SegmentGrowingImpl.h"

194
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -25,7 +25,7 @@ namespace milvus::query {
namespace impl {
class ExecExprVisitor : ExprVisitor {
public:
using RetType = std::deque<boost::dynamic_bitset<>>;
using RetType = boost::dynamic_bitset<>;
ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count, Timestamp timestamp)
: segment_(segment), row_count_(row_count), timestamp_(timestamp) {
}
@ -34,9 +34,9 @@ class ExecExprVisitor : ExprVisitor {
Assert(!ret_.has_value());
expr.accept(*this);
Assert(ret_.has_value());
auto ret = std::move(ret_);
auto res = std::move(ret_);
ret_ = std::nullopt;
return std::move(ret.value());
return std::move(res.value());
}
public:
@ -68,58 +68,71 @@ class ExecExprVisitor : ExprVisitor {
void
ExecExprVisitor::visit(LogicalUnaryExpr& expr) {
using OpType = LogicalUnaryExpr::OpType;
auto vec = call_child(*expr.child_);
RetType ret;
for (int chunk_id = 0; chunk_id < vec.size(); ++chunk_id) {
auto chunk = vec[chunk_id];
switch (expr.op_type_) {
case OpType::LogicalNot: {
chunk.flip();
break;
}
default: {
PanicInfo("Invalid OpType");
}
auto child_res = call_child(*expr.child_);
RetType res = std::move(child_res);
switch (expr.op_type_) {
case OpType::LogicalNot: {
res.flip();
break;
}
default: {
PanicInfo("Invalid Unary Op");
}
ret.emplace_back(std::move(chunk));
}
ret_ = std::move(ret);
Assert(res.size() == row_count_);
ret_ = std::move(res);
}
void
ExecExprVisitor::visit(LogicalBinaryExpr& expr) {
using OpType = LogicalBinaryExpr::OpType;
RetType ret;
auto left = call_child(*expr.left_);
auto right = call_child(*expr.right_);
Assert(left.size() == right.size());
for (int chunk_id = 0; chunk_id < left.size(); ++chunk_id) {
boost::dynamic_bitset<> chunk_res;
auto left_chunk = std::move(left[chunk_id]);
auto right_chunk = std::move(right[chunk_id]);
chunk_res = std::move(left_chunk);
switch (expr.op_type_) {
case OpType::LogicalAnd: {
chunk_res &= right_chunk;
break;
}
case OpType::LogicalOr: {
chunk_res |= right_chunk;
break;
}
case OpType::LogicalXor: {
chunk_res ^= right_chunk;
break;
}
case OpType::LogicalMinus: {
chunk_res -= right_chunk;
break;
}
auto res = std::move(left);
switch (expr.op_type_) {
case OpType::LogicalAnd: {
res &= right;
break;
}
case OpType::LogicalOr: {
res |= right;
break;
}
case OpType::LogicalXor: {
res ^= right;
break;
}
case OpType::LogicalMinus: {
res -= right;
break;
}
default: {
PanicInfo("Invalid Binary Op");
}
ret.emplace_back(std::move(chunk_res));
}
ret_ = std::move(ret);
Assert(res.size() == row_count_);
ret_ = std::move(res);
}
static auto
Assemble(const std::deque<boost::dynamic_bitset<>>& srcs) -> boost::dynamic_bitset<> {
boost::dynamic_bitset<> res;
int64_t total_size = 0;
for (auto& chunk : srcs) {
total_size += chunk.size();
}
res.resize(total_size);
int64_t counter = 0;
for (auto& chunk : srcs) {
for (int64_t i = 0; i < chunk.size(); ++i) {
res[counter + i] = chunk[i];
}
counter += chunk.size();
}
return res;
}
template <typename T, typename IndexFunc, typename ElementFunc>
@ -132,7 +145,7 @@ ExecExprVisitor::ExecRangeVisitorImpl(RangeExprImpl<T>& expr, IndexFunc index_fu
auto indexing_barrier = segment_.num_chunk_index(field_offset);
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
RetType results;
std::deque<boost::dynamic_bitset<>> results;
using Index = knowhere::scalar::StructuredIndex<T>;
for (auto chunk_id = 0; chunk_id < indexing_barrier; ++chunk_id) {
@ -143,20 +156,22 @@ ExecExprVisitor::ExecRangeVisitorImpl(RangeExprImpl<T>& expr, IndexFunc index_fu
Assert(data->size() == size_per_chunk);
results.emplace_back(std::move(*data));
}
for (auto chunk_id = indexing_barrier; chunk_id < num_chunk; ++chunk_id) {
boost::dynamic_bitset<> result(size_per_chunk);
result.resize(size_per_chunk);
auto this_size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
boost::dynamic_bitset<> result(this_size);
auto chunk = segment_.chunk_data<T>(field_offset, chunk_id);
const T* data = chunk.data();
for (int index = 0; index < size_per_chunk; ++index) {
for (int index = 0; index < this_size; ++index) {
result[index] = element_func(data[index]);
}
Assert(result.size() == size_per_chunk);
Assert(result.size() == this_size);
results.emplace_back(std::move(result));
}
return results;
auto final_result = Assemble(results);
Assert(final_result.size() == row_count_);
return final_result;
}
#pragma clang diagnostic push
#pragma ide diagnostic ignored "Simplify"
template <typename T>
@ -213,11 +228,8 @@ ExecExprVisitor::ExecRangeVisitorDispatcher(RangeExpr& expr_raw) -> RetType {
std::tie(op2, val2) = conditions[1];
// TODO: disable check?
if (val1 > val2) {
// Empty
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
RetType ret(num_chunk, boost::dynamic_bitset<>(size_per_chunk));
return ret;
RetType res(row_count_, false);
return res;
}
auto ops = std::make_tuple(op1, op2);
if (false) {
@ -246,40 +258,41 @@ void
ExecExprVisitor::visit(RangeExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_offset_];
Assert(expr.data_type_ == field_meta.get_data_type());
RetType ret;
RetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
ret = ExecRangeVisitorDispatcher<bool>(expr);
res = ExecRangeVisitorDispatcher<bool>(expr);
break;
}
case DataType::INT8: {
ret = ExecRangeVisitorDispatcher<int8_t>(expr);
res = ExecRangeVisitorDispatcher<int8_t>(expr);
break;
}
case DataType::INT16: {
ret = ExecRangeVisitorDispatcher<int16_t>(expr);
res = ExecRangeVisitorDispatcher<int16_t>(expr);
break;
}
case DataType::INT32: {
ret = ExecRangeVisitorDispatcher<int32_t>(expr);
res = ExecRangeVisitorDispatcher<int32_t>(expr);
break;
}
case DataType::INT64: {
ret = ExecRangeVisitorDispatcher<int64_t>(expr);
res = ExecRangeVisitorDispatcher<int64_t>(expr);
break;
}
case DataType::FLOAT: {
ret = ExecRangeVisitorDispatcher<float>(expr);
res = ExecRangeVisitorDispatcher<float>(expr);
break;
}
case DataType::DOUBLE: {
ret = ExecRangeVisitorDispatcher<double>(expr);
res = ExecRangeVisitorDispatcher<double>(expr);
break;
}
default:
PanicInfo("unsupported");
}
ret_ = std::move(ret);
Assert(res.size() == row_count_);
ret_ = std::move(res);
}
template <typename Op>
@ -303,10 +316,9 @@ auto
ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> RetType {
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
RetType bitsets;
std::deque<RetType> bitsets;
for (int64_t chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
auto getChunkData = [&, chunk_id](DataType type, FieldOffset offset) -> std::function<const number(int)> {
switch (type) {
case DataType::BOOL: {
@ -344,14 +356,16 @@ ExecExprVisitor::ExecCompareExprDispatcher(CompareExpr& expr, Op op) -> RetType
auto left = getChunkData(expr.data_types_[0], expr.field_offsets_[0]);
auto right = getChunkData(expr.data_types_[1], expr.field_offsets_[1]);
boost::dynamic_bitset<> bitset(size_per_chunk);
boost::dynamic_bitset<> bitset(size);
for (int i = 0; i < size; ++i) {
bool is_in = boost::apply_visitor(relational<decltype(op)>{}, left(i), right(i));
bitset[i] = is_in;
}
bitsets.emplace_back(std::move(bitset));
}
return bitsets;
auto final_result = Assemble(bitsets);
Assert(final_result.size() == row_count_);
return final_result;
}
void
@ -365,37 +379,38 @@ ExecExprVisitor::visit(CompareExpr& expr) {
Assert(expr.data_types_[i] == field_meta.get_data_type());
}
RetType ret;
RetType res;
switch (expr.op) {
case OpType::Equal: {
ret = ExecCompareExprDispatcher(expr, std::equal_to<>{});
res = ExecCompareExprDispatcher(expr, std::equal_to<>{});
break;
}
case OpType::NotEqual: {
ret = ExecCompareExprDispatcher(expr, std::not_equal_to<>{});
res = ExecCompareExprDispatcher(expr, std::not_equal_to<>{});
break;
}
case OpType::GreaterEqual: {
ret = ExecCompareExprDispatcher(expr, std::greater_equal<>{});
res = ExecCompareExprDispatcher(expr, std::greater_equal<>{});
break;
}
case OpType::GreaterThan: {
ret = ExecCompareExprDispatcher(expr, std::greater<>{});
res = ExecCompareExprDispatcher(expr, std::greater<>{});
break;
}
case OpType::LessEqual: {
ret = ExecCompareExprDispatcher(expr, std::less_equal<>{});
res = ExecCompareExprDispatcher(expr, std::less_equal<>{});
break;
}
case OpType::LessThan: {
ret = ExecCompareExprDispatcher(expr, std::less<>{});
res = ExecCompareExprDispatcher(expr, std::less<>{});
break;
}
default: {
PanicInfo("unsupported optype");
}
}
ret_ = std::move(ret);
Assert(res.size() == row_count_);
ret_ = std::move(res);
}
template <typename T>
@ -408,13 +423,13 @@ ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType {
auto& field_meta = schema[field_offset];
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
RetType bitsets;
std::deque<RetType> bitsets;
for (int64_t chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
Span<T> chunk = segment_.chunk_data<T>(field_offset, chunk_id);
auto size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
boost::dynamic_bitset<> bitset(size_per_chunk);
boost::dynamic_bitset<> bitset(size);
for (int i = 0; i < size; ++i) {
auto value = chunk.data()[i];
bool is_in = std::binary_search(expr.terms_.begin(), expr.terms_.end(), value);
@ -422,46 +437,49 @@ ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType {
}
bitsets.emplace_back(std::move(bitset));
}
return bitsets;
auto final_result = Assemble(bitsets);
Assert(final_result.size() == row_count_);
return final_result;
}
void
ExecExprVisitor::visit(TermExpr& expr) {
auto& field_meta = segment_.get_schema()[expr.field_offset_];
Assert(expr.data_type_ == field_meta.get_data_type());
RetType ret;
RetType res;
switch (expr.data_type_) {
case DataType::BOOL: {
ret = ExecTermVisitorImpl<bool>(expr);
res = ExecTermVisitorImpl<bool>(expr);
break;
}
case DataType::INT8: {
ret = ExecTermVisitorImpl<int8_t>(expr);
res = ExecTermVisitorImpl<int8_t>(expr);
break;
}
case DataType::INT16: {
ret = ExecTermVisitorImpl<int16_t>(expr);
res = ExecTermVisitorImpl<int16_t>(expr);
break;
}
case DataType::INT32: {
ret = ExecTermVisitorImpl<int32_t>(expr);
res = ExecTermVisitorImpl<int32_t>(expr);
break;
}
case DataType::INT64: {
ret = ExecTermVisitorImpl<int64_t>(expr);
res = ExecTermVisitorImpl<int64_t>(expr);
break;
}
case DataType::FLOAT: {
ret = ExecTermVisitorImpl<float>(expr);
res = ExecTermVisitorImpl<float>(expr);
break;
}
case DataType::DOUBLE: {
ret = ExecTermVisitorImpl<double>(expr);
res = ExecTermVisitorImpl<double>(expr);
break;
}
default:
PanicInfo("unsupported");
}
ret_ = std::move(ret);
Assert(res.size() == row_count_);
ret_ = std::move(res);
}
} // namespace milvus::query

13
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -18,6 +18,7 @@
#include "query/generated/ExecExprVisitor.h"
#include "query/SearchOnGrowing.h"
#include "query/SearchOnSealed.h"
#include "boost_ext/dynamic_bitset_ext.hpp"
namespace milvus::query {
@ -84,7 +85,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
auto src_data = ph.get_blob<EmbeddedType<VectorType>>();
auto num_queries = ph.num_of_queries_;
aligned_vector<uint8_t> bitset_holder;
boost::dynamic_bitset<> bitset_holder;
BitsetView view;
// TODO: add API to unify row_count
// auto row_count = segment->get_row_count();
@ -99,9 +100,13 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
if (node.predicate_.has_value()) {
ExecExprVisitor::RetType expr_ret =
ExecExprVisitor(*segment, active_count, timestamp_).call_child(*node.predicate_.value());
segment->mask_with_timestamps(expr_ret, timestamp_);
bitset_holder = AssembleNegBitset(expr_ret);
view = BitsetView(bitset_holder.data(), bitset_holder.size() * 8);
bitset_holder = std::move(expr_ret);
}
segment->mask_with_timestamps(bitset_holder, timestamp_);
if (!bitset_holder.empty()) {
bitset_holder.flip();
view = BitsetView((uint8_t*)boost_ext::get_data(bitset_holder), bitset_holder.size());
}
segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, view, ret);

3
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -484,8 +484,7 @@ SegmentGrowingImpl::get_active_count(Timestamp ts) const {
}
void
SegmentGrowingImpl::mask_with_timestamps(std::deque<boost::dynamic_bitset<>>& bitset_chunks,
Timestamp timestamp) const {
SegmentGrowingImpl::mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const {
// DO NOTHING
}

2
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -164,7 +164,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
}
void
mask_with_timestamps(std::deque<boost::dynamic_bitset<>>& bitset_chunks, Timestamp timestamp) const override;
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const override;
void
vector_search(int64_t vec_count,

2
internal/core/src/segcore/SegmentInterface.h
View File

@ -110,7 +110,7 @@ class SegmentInternalInterface : public SegmentInterface {
num_chunk_index(FieldOffset field_offset) const = 0;
virtual void
mask_with_timestamps(std::deque<boost::dynamic_bitset<>>& bitset_chunks, Timestamp timestamp) const = 0;
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const = 0;
// count of chunks
virtual int64_t

5
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -474,17 +474,16 @@ SegmentSealedImpl::get_active_count(Timestamp ts) const {
return this->get_row_count();
}
void
SegmentSealedImpl::mask_with_timestamps(std::deque<boost::dynamic_bitset<>>& bitset_chunks, Timestamp timestamp) const {
SegmentSealedImpl::mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const {
// TODO change the
Assert(this->timestamps_.size() == get_row_count());
Assert(bitset_chunks.size() == 1);
auto range = timestamp_index_.get_active_range(timestamp);
if (range.first == range.second && range.first == this->timestamps_.size()) {
// just skip
return;
}
auto mask = TimestampIndex::GenerateBitset(timestamp, range, this->timestamps_.data(), this->timestamps_.size());
bitset_chunks[0] &= mask;
bitset_chunk &= mask;
}
SegmentSealedPtr

2
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -107,7 +107,7 @@ class SegmentSealedImpl : public SegmentSealed {
}
void
mask_with_timestamps(std::deque<boost::dynamic_bitset<>>& bitset_chunks, Timestamp timestamp) const override;
mask_with_timestamps(boost::dynamic_bitset<>& bitset_chunk, Timestamp timestamp) const override;
void
vector_search(int64_t vec_count,

54
internal/core/unittest/test_c_api.cpp
View File

@ -341,7 +341,7 @@ TEST(CApiTest, GetMemoryUsageInBytesTest) {
auto segment = NewSegment(collection, 0, Growing);
auto old_memory_usage_size = GetMemoryUsageInBytes(segment);
//std::cout << "old_memory_usage_size = " << old_memory_usage_size << std::endl;
// std::cout << "old_memory_usage_size = " << old_memory_usage_size << std::endl;
assert(old_memory_usage_size == 0);
int N = 10000;
@ -355,7 +355,7 @@ TEST(CApiTest, GetMemoryUsageInBytesTest) {
assert(res.error_code == Success);
auto memory_usage_size = GetMemoryUsageInBytes(segment);
//std::cout << "new_memory_usage_size = " << memory_usage_size << std::endl;
// std::cout << "new_memory_usage_size = " << memory_usage_size << std::endl;
assert(memory_usage_size == 2785280);
DeleteCollection(collection);
@ -697,7 +697,7 @@ TEST(CApiTest, LoadIndex_Search) {
auto ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result->Get<float*>(milvus::knowhere::meta::DISTANCE);
//for (int i = 0; i < std::min(num_query * K, 100); ++i) {
// for (int i = 0; i < std::min(num_query * K, 100); ++i) {
// std::cout << ids[i] << "->" << dis[i] << std::endl;
//}
}
@ -718,8 +718,8 @@ TEST(CApiTest, Indexing_Without_Predicate) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* dsl_string = R"(
@ -811,8 +811,8 @@ TEST(CApiTest, Indexing_Without_Predicate) {
auto search_result_on_raw_index_json = SearchResultToJson(*search_result_on_raw_index);
auto search_result_on_bigIndex_json = SearchResultToJson((*(SearchResult*)c_search_result_on_bigIndex));
//std::cout << search_result_on_raw_index_json.dump(1) << std::endl;
//std::cout << search_result_on_bigIndex_json.dump(1) << std::endl;
// std::cout << search_result_on_raw_index_json.dump(1) << std::endl;
// std::cout << search_result_on_bigIndex_json.dump(1) << std::endl;
ASSERT_EQ(search_result_on_raw_index_json.dump(1), search_result_on_bigIndex_json.dump(1));
@ -841,8 +841,8 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* serialized_expr_plan = R"(vector_anns: <
@ -929,8 +929,8 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
auto search_result_on_raw_index_json = SearchResultToJson(*search_result_on_raw_index);
auto search_result_on_bigIndex_json = SearchResultToJson((*(SearchResult*)c_search_result_on_bigIndex));
//std::cout << search_result_on_raw_index_json.dump(1) << std::endl;
//std::cout << search_result_on_bigIndex_json.dump(1) << std::endl;
// std::cout << search_result_on_raw_index_json.dump(1) << std::endl;
// std::cout << search_result_on_bigIndex_json.dump(1) << std::endl;
ASSERT_EQ(search_result_on_raw_index_json.dump(1), search_result_on_bigIndex_json.dump(1));
@ -959,8 +959,8 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* dsl_string = R"({
@ -1096,8 +1096,8 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
{
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
}
@ -1246,8 +1246,8 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* dsl_string = R"({
@ -1381,8 +1381,8 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* serialized_expr_plan = R"(vector_anns: <
@ -1581,8 +1581,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* dsl_string = R"({
@ -1718,8 +1718,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* serialized_expr_plan = R"(vector_anns: <
@ -1868,8 +1868,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* dsl_string = R"({
@ -2011,8 +2011,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
int64_t offset;
PreInsert(segment, N, &offset);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data,
dataset.raw_.sizeof_per_row, dataset.raw_.count);
auto ins_res = Insert(segment, offset, N, dataset.row_ids_.data(), dataset.timestamps_.data(),
dataset.raw_.raw_data, dataset.raw_.sizeof_per_row, dataset.raw_.count);
assert(ins_res.error_code == Success);
const char* serialized_expr_plan = R"(vector_anns: <

24
internal/core/unittest/test_expr.cpp
View File

@ -315,12 +315,10 @@ TEST(Expr, TestRange) {
dsl_string.replace(loc, 4, clause);
auto plan = CreatePlan(*schema, dsl_string);
auto final = visitor.call_child(*plan->plan_node_->predicate_.value());
EXPECT_EQ(final.size(), upper_div(N * num_iters, TestChunkSize));
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto vec_id = i / TestChunkSize;
auto offset = i % TestChunkSize;
auto ans = final[vec_id][offset];
auto ans = final[i];
auto val = age_col[i];
auto ref = ref_func(val);
@ -399,12 +397,10 @@ TEST(Expr, TestTerm) {
dsl_string.replace(loc, 4, clause);
auto plan = CreatePlan(*schema, dsl_string);
auto final = visitor.call_child(*plan->plan_node_->predicate_.value());
EXPECT_EQ(final.size(), upper_div(N * num_iters, TestChunkSize));
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto vec_id = i / TestChunkSize;
auto offset = i % TestChunkSize;
auto ans = final[vec_id][offset];
auto ans = final[i];
auto val = age_col[i];
auto ref = ref_func(val);
@ -501,12 +497,10 @@ TEST(Expr, TestSimpleDsl) {
// std::cout << dsl.dump(2);
auto plan = CreatePlan(*schema, dsl.dump());
auto final = visitor.call_child(*plan->plan_node_->predicate_.value());
EXPECT_EQ(final.size(), upper_div(N * num_iters, TestChunkSize));
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto vec_id = i / TestChunkSize;
auto offset = i % TestChunkSize;
bool ans = final[vec_id][offset];
bool ans = final[i];
auto val = age_col[i];
auto ref = ref_func(val);
ASSERT_EQ(ans, ref) << clause << "@" << i << "!!" << val;
@ -577,12 +571,10 @@ TEST(Expr, TestCompare) {
auto plan = CreatePlan(*schema, dsl_string);
// std::cout << ShowPlanNodeVisitor().call_child(*plan->plan_node_) << std::endl;
auto final = visitor.call_child(*plan->plan_node_->predicate_.value());
EXPECT_EQ(final.size(), upper_div(N * num_iters, TestChunkSize));
EXPECT_EQ(final.size(), N * num_iters);
for (int i = 0; i < N * num_iters; ++i) {
auto vec_id = i / TestChunkSize;
auto offset = i % TestChunkSize;
auto ans = final[vec_id][offset];
auto ans = final[i];
auto val1 = age1_col[i];
auto val2 = age2_col[i];