mirror of https://github.com/milvus-io/milvus.git
Create Jenkins pipeline
Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>pull/4973/head^2
quicksilver
yefu.chen
parent
77fa75b1ec
commit
86573d0053
8
Makefile
8
Makefile
|
|
@ -55,13 +55,13 @@ verifiers: cppcheck fmt lint ruleguard
|
|||
|
||||
# Builds various components locally.
|
||||
build-go:
|
||||
@echo "Building each component's binary to './bin'"
|
||||
@echo "Building each component's binary to './'"
|
||||
@echo "Building query node ..."
|
||||
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/querynode $(PWD)/cmd/querynode/query_node.go 1>/dev/null
|
||||
@mkdir -p $(INSTALL_PATH) && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/querynode $(PWD)/cmd/querynode/query_node.go 1>/dev/null
|
||||
@echo "Building master ..."
|
||||
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="0" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/master $(PWD)/cmd/master/main.go 1>/dev/null
|
||||
@mkdir -p $(INSTALL_PATH) && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/master $(PWD)/cmd/master/main.go 1>/dev/null
|
||||
@echo "Building proxy ..."
|
||||
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="0" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/proxy $(PWD)/cmd/proxy/proxy.go 1>/dev/null
|
||||
@mkdir -p $(INSTALL_PATH) && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/proxy $(PWD)/cmd/proxy/proxy.go 1>/dev/null
|
||||
|
||||
build-cpp:
|
||||
@(env bash $(PWD)/scripts/core_build.sh)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ pipeline {
|
|||
stage ('Build and UnitTest') {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-build"
|
||||
label "${env.PROJECT_NAME}-${env.BUILD_NUMBER}-build"
|
||||
defaultContainer 'build-env'
|
||||
customWorkspace '/home/jenkins/agent/workspace'
|
||||
yamlFile "build/ci/jenkins/pod/build-env.yaml"
|
||||
|
|
@ -46,7 +46,7 @@ pipeline {
|
|||
stage ('Publish Docker Images') {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-publish"
|
||||
label "${env.PROJECT_NAME}-${env.BUILD_NUMBER}-publish"
|
||||
defaultContainer 'publish-images'
|
||||
yamlFile "build/ci/jenkins/pod/docker-pod.yaml"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ services:
|
|||
environment:
|
||||
PULSAR_ADDRESS: ${PULSAR_ADDRESS}
|
||||
ETCD_ADDRESS: ${ETCD_ADDRESS}
|
||||
MASTER_ADDRESS: ${MASTER_ADDRESS}
|
||||
networks:
|
||||
- milvus
|
||||
ports:
|
||||
|
|
@ -25,6 +26,7 @@ services:
|
|||
- ${SOURCE_REPO}/proxy:${SOURCE_TAG}
|
||||
environment:
|
||||
PULSAR_ADDRESS: ${PULSAR_ADDRESS}
|
||||
ETCD_ADDRESS: ${ETCD_ADDRESS}
|
||||
MASTER_ADDRESS: ${MASTER_ADDRESS}
|
||||
ports:
|
||||
- "19530:19530"
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ func (sa *SegIDAssigner) syncSegments() {
|
|||
resp, err := sa.masterClient.AssignSegmentID(ctx, req)
|
||||
|
||||
if resp.Status.GetErrorCode() != commonpb.ErrorCode_SUCCESS {
|
||||
log.Println("GRPC AssignSegmentID Failed", resp, err)
|
||||
log.Panic("GRPC AssignSegmentID Failed")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
LICENSE=$1
|
||||
FOLDER=$2
|
||||
|
||||
if [ -z ${FOLDER} ] || [ -z ${LICENSE} ]; then
|
||||
echo "usage $0 <path/to/license> <path/to/code_folder>"
|
||||
FOLDER=$1
|
||||
if [ -z ${FOLDER} ]; then
|
||||
echo usage $0 [folder_to_add_license]
|
||||
exit
|
||||
else
|
||||
echo good
|
||||
fi
|
||||
|
||||
cat ${LICENSE} > /dev/null || exit -1
|
||||
|
||||
FILES=`find ${FOLDER} \
|
||||
| grep -E "(*\.cpp$|*\.h$|*\.cu$)" \
|
||||
| grep -v thirdparty \
|
||||
|
|
@ -15,16 +13,13 @@ FILES=`find ${FOLDER} \
|
|||
| grep -v cmake-build \
|
||||
| grep -v output \
|
||||
| grep -v "\.pb\."`
|
||||
# echo formating ${FILES} ...
|
||||
skip_count=0
|
||||
echo formating ${FILES} ...
|
||||
for f in ${FILES}; do
|
||||
if (grep "Apache License" $f > /dev/null);then
|
||||
# echo "No need to copy the License Header to $f"
|
||||
skip_count=$((skip_count+1))
|
||||
if (grep "Apache License" $f);then
|
||||
echo "No need to copy the License Header to $f"
|
||||
else
|
||||
cat ${LICENSE} $f > $f.new
|
||||
cat cpp_license.txt $f > $f.new
|
||||
mv $f.new $f
|
||||
echo "License Header copied to $f"
|
||||
fi
|
||||
done
|
||||
echo "license adder: $skip_count file(s) skiped"
|
||||
|
|
|
|||
|
|
@ -13,4 +13,3 @@ formatThis() {
|
|||
formatThis "${CorePath}/src"
|
||||
formatThis "${CorePath}/unittest"
|
||||
|
||||
${CorePath}/build-support/add_license.sh ${CorePath}/build-support/cpp_license.txt ${CorePath}
|
||||
|
|
|
|||
|
|
@ -473,7 +473,7 @@ const char descriptor_table_protodef_service_5fmsg_2eproto[] PROTOBUF_SECTION_VA
|
|||
"g\030\002 \001(\t\"z\n\010RowBatch\022\027\n\017collection_name\030\001"
|
||||
" \001(\t\022\025\n\rpartition_tag\030\002 \001(\t\022+\n\010row_data\030"
|
||||
"\003 \003(\0132\031.milvus.proto.common.Blob\022\021\n\thash"
|
||||
"_keys\030\004 \003(\r\"d\n\020PlaceholderValue\022\013\n\003tag\030\001"
|
||||
"_keys\030\004 \003(\005\"d\n\020PlaceholderValue\022\013\n\003tag\030\001"
|
||||
" \001(\t\0223\n\004type\030\002 \001(\0162%.milvus.proto.servic"
|
||||
"e.PlaceholderType\022\016\n\006values\030\003 \003(\014\"P\n\020Pla"
|
||||
"ceholderGroup\022<\n\014placeholders\030\001 \003(\0132&.mi"
|
||||
|
|
@ -1265,10 +1265,10 @@ const char* RowBatch::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::i
|
|||
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 26);
|
||||
} else goto handle_unusual;
|
||||
continue;
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
case 4:
|
||||
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) {
|
||||
ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedUInt32Parser(mutable_hash_keys(), ptr, ctx);
|
||||
ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(mutable_hash_keys(), ptr, ctx);
|
||||
CHK_(ptr);
|
||||
} else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32) {
|
||||
add_hash_keys(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr));
|
||||
|
|
@ -1346,15 +1346,15 @@ bool RowBatch::MergePartialFromCodedStream(
|
|||
break;
|
||||
}
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
case 4: {
|
||||
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (34 & 0xFF)) {
|
||||
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive<
|
||||
::PROTOBUF_NAMESPACE_ID::uint32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_UINT32>(
|
||||
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
|
||||
input, this->mutable_hash_keys())));
|
||||
} else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) {
|
||||
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
|
||||
::PROTOBUF_NAMESPACE_ID::uint32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_UINT32>(
|
||||
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
|
||||
1, 34u, input, this->mutable_hash_keys())));
|
||||
} else {
|
||||
goto handle_unusual;
|
||||
|
|
@ -1418,14 +1418,14 @@ void RowBatch::SerializeWithCachedSizes(
|
|||
output);
|
||||
}
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
if (this->hash_keys_size() > 0) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(4, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
|
||||
output->WriteVarint32(_hash_keys_cached_byte_size_.load(
|
||||
std::memory_order_relaxed));
|
||||
}
|
||||
for (int i = 0, n = this->hash_keys_size(); i < n; i++) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt32NoTag(
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32NoTag(
|
||||
this->hash_keys(i), output);
|
||||
}
|
||||
|
||||
|
|
@ -1472,7 +1472,7 @@ void RowBatch::SerializeWithCachedSizes(
|
|||
3, this->row_data(static_cast<int>(i)), target);
|
||||
}
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
if (this->hash_keys_size() > 0) {
|
||||
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray(
|
||||
4,
|
||||
|
|
@ -1482,7 +1482,7 @@ void RowBatch::SerializeWithCachedSizes(
|
|||
_hash_keys_cached_byte_size_.load(std::memory_order_relaxed),
|
||||
target);
|
||||
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
|
||||
WriteUInt32NoTagToArray(this->hash_keys_, target);
|
||||
WriteInt32NoTagToArray(this->hash_keys_, target);
|
||||
}
|
||||
|
||||
if (_internal_metadata_.have_unknown_fields()) {
|
||||
|
|
@ -1517,10 +1517,10 @@ size_t RowBatch::ByteSizeLong() const {
|
|||
}
|
||||
}
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
{
|
||||
size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
|
||||
UInt32Size(this->hash_keys_);
|
||||
Int32Size(this->hash_keys_);
|
||||
if (data_size > 0) {
|
||||
total_size += 1 +
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
|
||||
|
|
|
|||
|
|
@ -573,15 +573,15 @@ class RowBatch :
|
|||
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob >&
|
||||
row_data() const;
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
int hash_keys_size() const;
|
||||
void clear_hash_keys();
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 hash_keys(int index) const;
|
||||
void set_hash_keys(int index, ::PROTOBUF_NAMESPACE_ID::uint32 value);
|
||||
void add_hash_keys(::PROTOBUF_NAMESPACE_ID::uint32 value);
|
||||
const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::uint32 >&
|
||||
::PROTOBUF_NAMESPACE_ID::int32 hash_keys(int index) const;
|
||||
void set_hash_keys(int index, ::PROTOBUF_NAMESPACE_ID::int32 value);
|
||||
void add_hash_keys(::PROTOBUF_NAMESPACE_ID::int32 value);
|
||||
const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int32 >&
|
||||
hash_keys() const;
|
||||
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::uint32 >*
|
||||
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int32 >*
|
||||
mutable_hash_keys();
|
||||
|
||||
// string collection_name = 1;
|
||||
|
|
@ -612,7 +612,7 @@ class RowBatch :
|
|||
|
||||
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
|
||||
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob > row_data_;
|
||||
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::uint32 > hash_keys_;
|
||||
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int32 > hash_keys_;
|
||||
mutable std::atomic<int> _hash_keys_cached_byte_size_;
|
||||
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr collection_name_;
|
||||
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_;
|
||||
|
|
@ -2777,31 +2777,31 @@ RowBatch::row_data() const {
|
|||
return row_data_;
|
||||
}
|
||||
|
||||
// repeated uint32 hash_keys = 4;
|
||||
// repeated int32 hash_keys = 4;
|
||||
inline int RowBatch::hash_keys_size() const {
|
||||
return hash_keys_.size();
|
||||
}
|
||||
inline void RowBatch::clear_hash_keys() {
|
||||
hash_keys_.Clear();
|
||||
}
|
||||
inline ::PROTOBUF_NAMESPACE_ID::uint32 RowBatch::hash_keys(int index) const {
|
||||
inline ::PROTOBUF_NAMESPACE_ID::int32 RowBatch::hash_keys(int index) const {
|
||||
// @@protoc_insertion_point(field_get:milvus.proto.service.RowBatch.hash_keys)
|
||||
return hash_keys_.Get(index);
|
||||
}
|
||||
inline void RowBatch::set_hash_keys(int index, ::PROTOBUF_NAMESPACE_ID::uint32 value) {
|
||||
inline void RowBatch::set_hash_keys(int index, ::PROTOBUF_NAMESPACE_ID::int32 value) {
|
||||
hash_keys_.Set(index, value);
|
||||
// @@protoc_insertion_point(field_set:milvus.proto.service.RowBatch.hash_keys)
|
||||
}
|
||||
inline void RowBatch::add_hash_keys(::PROTOBUF_NAMESPACE_ID::uint32 value) {
|
||||
inline void RowBatch::add_hash_keys(::PROTOBUF_NAMESPACE_ID::int32 value) {
|
||||
hash_keys_.Add(value);
|
||||
// @@protoc_insertion_point(field_add:milvus.proto.service.RowBatch.hash_keys)
|
||||
}
|
||||
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::uint32 >&
|
||||
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int32 >&
|
||||
RowBatch::hash_keys() const {
|
||||
// @@protoc_insertion_point(field_list:milvus.proto.service.RowBatch.hash_keys)
|
||||
return hash_keys_;
|
||||
}
|
||||
inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::uint32 >*
|
||||
inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int32 >*
|
||||
RowBatch::mutable_hash_keys() {
|
||||
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.RowBatch.hash_keys)
|
||||
return &hash_keys_;
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include "BruteForceSearch.h"
|
||||
#include <vector>
|
||||
|
||||
namespace milvus::query {
|
||||
|
||||
void
|
||||
BinarySearchBruteForce(faiss::MetricType metric_type,
|
||||
int64_t code_size,
|
||||
const uint8_t* binary_chunk,
|
||||
int64_t chunk_size,
|
||||
int64_t topk,
|
||||
int64_t num_queries,
|
||||
const uint8_t* query_data,
|
||||
float* result_distances,
|
||||
idx_t* result_labels,
|
||||
faiss::ConcurrentBitsetPtr bitset) {
|
||||
const idx_t block_size = segcore::DefaultElementPerChunk;
|
||||
bool use_heap = true;
|
||||
|
||||
if (metric_type == faiss::METRIC_Jaccard || metric_type == faiss::METRIC_Tanimoto) {
|
||||
float* D = result_distances;
|
||||
for (idx_t query_base_index = 0; query_base_index < num_queries; query_base_index += block_size) {
|
||||
idx_t query_size = block_size;
|
||||
if (query_base_index + block_size > num_queries) {
|
||||
query_size = num_queries - query_base_index;
|
||||
}
|
||||
|
||||
// We see the distances and labels as heaps.
|
||||
faiss::float_maxheap_array_t res = {size_t(query_size), size_t(topk),
|
||||
result_labels + query_base_index * topk, D + query_base_index * topk};
|
||||
|
||||
binary_distence_knn_hc(metric_type, &res, query_data + query_base_index * code_size, binary_chunk,
|
||||
chunk_size, code_size,
|
||||
/* ordered = */ true, bitset);
|
||||
}
|
||||
if (metric_type == faiss::METRIC_Tanimoto) {
|
||||
for (int i = 0; i < topk * num_queries; i++) {
|
||||
D[i] = -log2(1 - D[i]);
|
||||
}
|
||||
}
|
||||
} else if (metric_type == faiss::METRIC_Substructure || metric_type == faiss::METRIC_Superstructure) {
|
||||
float* D = result_distances;
|
||||
for (idx_t s = 0; s < num_queries; s += block_size) {
|
||||
idx_t nn = block_size;
|
||||
if (s + block_size > num_queries) {
|
||||
nn = num_queries - s;
|
||||
}
|
||||
|
||||
// only match ids will be chosed, not to use heap
|
||||
binary_distence_knn_mc(metric_type, query_data + s * code_size, binary_chunk, nn, chunk_size, topk,
|
||||
code_size, D + s * topk, result_labels + s * topk, bitset);
|
||||
}
|
||||
} else if (metric_type == faiss::METRIC_Hamming) {
|
||||
std::vector<int> int_distances(topk * num_queries);
|
||||
for (idx_t s = 0; s < num_queries; s += block_size) {
|
||||
idx_t nn = block_size;
|
||||
if (s + block_size > num_queries) {
|
||||
nn = num_queries - s;
|
||||
}
|
||||
if (use_heap) {
|
||||
// We see the distances and labels as heaps.
|
||||
faiss::int_maxheap_array_t res = {size_t(nn), size_t(topk), result_labels + s * topk,
|
||||
int_distances.data() + s * topk};
|
||||
|
||||
hammings_knn_hc(&res, query_data + s * code_size, binary_chunk, chunk_size, code_size,
|
||||
/* ordered = */ true, bitset);
|
||||
} else {
|
||||
hammings_knn_mc(query_data + s * code_size, binary_chunk, nn, chunk_size, topk, code_size,
|
||||
int_distances.data() + s * topk, result_labels + s * topk, bitset);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < num_queries; ++i) {
|
||||
result_distances[i] = static_cast<float>(int_distances[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace milvus::query
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#pragma once
|
||||
#include <faiss/utils/BinaryDistance.h>
|
||||
#include "segcore/ConcurrentVector.h"
|
||||
#include "common/Schema.h"
|
||||
|
||||
namespace milvus::query {
|
||||
void
|
||||
BinarySearchBruteForce(faiss::MetricType metric_type,
|
||||
int64_t code_size,
|
||||
const uint8_t* binary_chunk,
|
||||
int64_t chunk_size,
|
||||
int64_t topk,
|
||||
int64_t num_queries,
|
||||
const uint8_t* query_data,
|
||||
float* result_distances,
|
||||
idx_t* result_labels,
|
||||
faiss::ConcurrentBitsetPtr bitset = nullptr);
|
||||
} // namespace milvus::query
|
||||
|
|
@ -9,7 +9,6 @@ set(MILVUS_QUERY_SRCS
|
|||
visitors/ExecExprVisitor.cpp
|
||||
Plan.cpp
|
||||
Search.cpp
|
||||
BruteForceSearch.cpp
|
||||
)
|
||||
add_library(milvus_query ${MILVUS_QUERY_SRCS})
|
||||
target_link_libraries(milvus_query milvus_proto)
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
|
|||
}
|
||||
segcore::merge_into(num_queries, topK, final_dis.data(), final_uids.data(), dis, uids);
|
||||
}
|
||||
using segcore::FloatVector;
|
||||
auto vec_ptr = record.get_entity<FloatVector>(vecfield_offset);
|
||||
|
||||
auto vec_ptr = record.get_vec_entity<float>(vecfield_offset);
|
||||
// step 4: brute force search where small indexing is unavailable
|
||||
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
|
||||
std::vector<int64_t> buf_uids(total_count, -1);
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ ExecExprVisitor::ExecRangeVisitorImpl(RangeExprImpl<T>& expr, IndexFunc index_fu
|
|||
Assert(field_offset_opt);
|
||||
auto field_offset = field_offset_opt.value();
|
||||
auto& field_meta = schema[field_offset];
|
||||
auto vec_ptr = records.get_entity<T>(field_offset);
|
||||
auto vec_ptr = records.get_scalar_entity<T>(field_offset);
|
||||
auto& vec = *vec_ptr;
|
||||
auto& indexing_record = segment_.get_indexing_record();
|
||||
const segcore::ScalarIndexingEntry<T>& entry = indexing_record.get_scalar_entry<T>(field_offset);
|
||||
|
|
|
|||
|
|
@ -109,20 +109,20 @@ class VectorBase {
|
|||
};
|
||||
|
||||
template <typename Type, bool is_scalar = false, ssize_t ElementsPerChunk = DefaultElementPerChunk>
|
||||
class ConcurrentVectorImpl : public VectorBase {
|
||||
class ConcurrentVector : public VectorBase {
|
||||
public:
|
||||
// constants
|
||||
using Chunk = FixedVector<Type>;
|
||||
ConcurrentVectorImpl(ConcurrentVectorImpl&&) = delete;
|
||||
ConcurrentVectorImpl(const ConcurrentVectorImpl&) = delete;
|
||||
ConcurrentVector(ConcurrentVector&&) = delete;
|
||||
ConcurrentVector(const ConcurrentVector&) = delete;
|
||||
|
||||
ConcurrentVectorImpl&
|
||||
operator=(ConcurrentVectorImpl&&) = delete;
|
||||
ConcurrentVectorImpl&
|
||||
operator=(const ConcurrentVectorImpl&) = delete;
|
||||
ConcurrentVector&
|
||||
operator=(ConcurrentVector&&) = delete;
|
||||
ConcurrentVector&
|
||||
operator=(const ConcurrentVector&) = delete;
|
||||
|
||||
public:
|
||||
explicit ConcurrentVectorImpl(ssize_t dim = 1) : Dim(is_scalar ? 1 : dim), SizePerChunk(Dim * ElementsPerChunk) {
|
||||
explicit ConcurrentVector(ssize_t dim = 1) : Dim(is_scalar ? 1 : dim), SizePerChunk(Dim * ElementsPerChunk) {
|
||||
Assert(is_scalar ? dim == 1 : dim != 1);
|
||||
}
|
||||
|
||||
|
|
@ -221,28 +221,4 @@ class ConcurrentVectorImpl : public VectorBase {
|
|||
ThreadSafeVector<Chunk> chunks_;
|
||||
};
|
||||
|
||||
template <typename Type>
|
||||
class ConcurrentVector : public ConcurrentVectorImpl<Type, true> {
|
||||
using ConcurrentVectorImpl<Type, true>::ConcurrentVectorImpl;
|
||||
};
|
||||
|
||||
class FloatVector {};
|
||||
class BinaryVector {};
|
||||
|
||||
template <>
|
||||
class ConcurrentVector<FloatVector> : public ConcurrentVectorImpl<float, false> {
|
||||
using ConcurrentVectorImpl<float, false>::ConcurrentVectorImpl;
|
||||
};
|
||||
|
||||
template <>
|
||||
class ConcurrentVector<BinaryVector> : public ConcurrentVectorImpl<uint8_t, false> {
|
||||
public:
|
||||
explicit ConcurrentVector(int64_t dim) : binary_dim_(dim), ConcurrentVectorImpl(dim / 8) {
|
||||
Assert(dim % 8 == 0);
|
||||
}
|
||||
|
||||
private:
|
||||
int64_t binary_dim_;
|
||||
};
|
||||
|
||||
} // namespace milvus::segcore
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ struct DeletedRecord {
|
|||
public:
|
||||
std::atomic<int64_t> reserved = 0;
|
||||
AckResponder ack_responder_;
|
||||
ConcurrentVector<Timestamp> timestamps_;
|
||||
ConcurrentVector<idx_t> uids_;
|
||||
ConcurrentVector<Timestamp, true> timestamps_;
|
||||
ConcurrentVector<idx_t, true> uids_;
|
||||
|
||||
private:
|
||||
std::shared_ptr<TmpBitmap> lru_;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ VecIndexingEntry::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vector
|
|||
assert(field_meta_.get_data_type() == DataType::VECTOR_FLOAT);
|
||||
auto dim = field_meta_.get_dim();
|
||||
|
||||
auto source = dynamic_cast<const ConcurrentVector<FloatVector>*>(vec_base);
|
||||
auto source = dynamic_cast<const ConcurrentVector<float>*>(vec_base);
|
||||
Assert(source);
|
||||
auto chunk_size = source->chunk_size();
|
||||
assert(ack_end <= chunk_size);
|
||||
|
|
@ -87,7 +87,7 @@ void
|
|||
ScalarIndexingEntry<T>::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
|
||||
auto dim = field_meta_.get_dim();
|
||||
|
||||
auto source = dynamic_cast<const ConcurrentVector<T>*>(vec_base);
|
||||
auto source = dynamic_cast<const ConcurrentVector<T, true>*>(vec_base);
|
||||
Assert(source);
|
||||
auto chunk_size = source->chunk_size();
|
||||
assert(ack_end <= chunk_size);
|
||||
|
|
@ -106,12 +106,7 @@ ScalarIndexingEntry<T>::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const
|
|||
std::unique_ptr<IndexingEntry>
|
||||
CreateIndex(const FieldMeta& field_meta) {
|
||||
if (field_meta.is_vector()) {
|
||||
if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) {
|
||||
return std::make_unique<VecIndexingEntry>(field_meta);
|
||||
} else {
|
||||
// TODO
|
||||
PanicInfo("unsupported");
|
||||
}
|
||||
return std::make_unique<VecIndexingEntry>(field_meta);
|
||||
}
|
||||
switch (field_meta.get_data_type()) {
|
||||
case DataType::INT8:
|
||||
|
|
|
|||
|
|
@ -100,9 +100,7 @@ class IndexingRecord {
|
|||
Initialize() {
|
||||
int offset = 0;
|
||||
for (auto& field : schema_) {
|
||||
if (field.get_data_type() != DataType::VECTOR_BINARY) {
|
||||
entries_.try_emplace(offset, CreateIndex(field));
|
||||
}
|
||||
entries_.try_emplace(offset, CreateIndex(field));
|
||||
++offset;
|
||||
}
|
||||
assert(offset == schema_.size());
|
||||
|
|
|
|||
|
|
@ -16,42 +16,36 @@ namespace milvus::segcore {
|
|||
InsertRecord::InsertRecord(const Schema& schema) : uids_(1), timestamps_(1) {
|
||||
for (auto& field : schema) {
|
||||
if (field.is_vector()) {
|
||||
if (field.get_data_type() == DataType::VECTOR_FLOAT) {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<FloatVector>>(field.get_dim()));
|
||||
continue;
|
||||
} else if (field.get_data_type() == DataType::VECTOR_BINARY) {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<BinaryVector>>(field.get_dim()));
|
||||
continue;
|
||||
} else {
|
||||
PanicInfo("unsupported");
|
||||
}
|
||||
Assert(field.get_data_type() == DataType::VECTOR_FLOAT);
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<float>>(field.get_dim()));
|
||||
continue;
|
||||
}
|
||||
switch (field.get_data_type()) {
|
||||
case DataType::INT8: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int8_t>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int8_t, true>>());
|
||||
break;
|
||||
}
|
||||
case DataType::INT16: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int16_t>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int16_t, true>>());
|
||||
break;
|
||||
}
|
||||
case DataType::INT32: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int32_t>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int32_t, true>>());
|
||||
break;
|
||||
}
|
||||
|
||||
case DataType::INT64: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int64_t>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<int64_t, true>>());
|
||||
break;
|
||||
}
|
||||
|
||||
case DataType::FLOAT: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<float>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<float, true>>());
|
||||
break;
|
||||
}
|
||||
|
||||
case DataType::DOUBLE: {
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<double>>());
|
||||
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<double, true>>());
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
|
|
|
|||
|
|
@ -21,14 +21,22 @@ namespace milvus::segcore {
|
|||
struct InsertRecord {
|
||||
std::atomic<int64_t> reserved = 0;
|
||||
AckResponder ack_responder_;
|
||||
ConcurrentVector<Timestamp> timestamps_;
|
||||
ConcurrentVector<idx_t> uids_;
|
||||
ConcurrentVector<Timestamp, true> timestamps_;
|
||||
ConcurrentVector<idx_t, true> uids_;
|
||||
std::vector<std::shared_ptr<VectorBase>> entity_vec_;
|
||||
|
||||
explicit InsertRecord(const Schema& schema);
|
||||
template <typename Type>
|
||||
auto
|
||||
get_entity(int offset) const {
|
||||
get_scalar_entity(int offset) const {
|
||||
auto ptr = std::dynamic_pointer_cast<const ConcurrentVector<Type, true>>(entity_vec_[offset]);
|
||||
Assert(ptr);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
auto
|
||||
get_vec_entity(int offset) const {
|
||||
auto ptr = std::dynamic_pointer_cast<const ConcurrentVector<Type>>(entity_vec_[offset]);
|
||||
Assert(ptr);
|
||||
return ptr;
|
||||
|
|
@ -36,7 +44,15 @@ struct InsertRecord {
|
|||
|
||||
template <typename Type>
|
||||
auto
|
||||
get_entity(int offset) {
|
||||
get_scalar_entity(int offset) {
|
||||
auto ptr = std::dynamic_pointer_cast<ConcurrentVector<Type, true>>(entity_vec_[offset]);
|
||||
Assert(ptr);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
auto
|
||||
get_vec_entity(int offset) {
|
||||
auto ptr = std::dynamic_pointer_cast<ConcurrentVector<Type>>(entity_vec_[offset]);
|
||||
Assert(ptr);
|
||||
return ptr;
|
||||
|
|
|
|||
|
|
@ -249,8 +249,7 @@ SegmentNaive::QueryImpl(query::QueryDeprecatedPtr query_info, Timestamp timestam
|
|||
auto the_offset_opt = schema_->get_offset(query_info->field_name);
|
||||
Assert(the_offset_opt.has_value());
|
||||
Assert(the_offset_opt.value() < record_.entity_vec_.size());
|
||||
auto vec_ptr =
|
||||
std::static_pointer_cast<ConcurrentVector<FloatVector>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
auto vec_ptr = std::static_pointer_cast<ConcurrentVector<float>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
auto index_entry = index_meta_->lookup_by_field(query_info->field_name);
|
||||
auto conf = index_entry.config;
|
||||
|
||||
|
|
@ -309,8 +308,7 @@ SegmentNaive::QueryBruteForceImpl(query::QueryDeprecatedPtr query_info, Timestam
|
|||
auto the_offset_opt = schema_->get_offset(query_info->field_name);
|
||||
Assert(the_offset_opt.has_value());
|
||||
Assert(the_offset_opt.value() < record_.entity_vec_.size());
|
||||
auto vec_ptr =
|
||||
std::static_pointer_cast<ConcurrentVector<FloatVector>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
auto vec_ptr = std::static_pointer_cast<ConcurrentVector<float>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
|
||||
std::vector<int64_t> final_uids(total_count);
|
||||
std::vector<float> final_dis(total_count, std::numeric_limits<float>::max());
|
||||
|
|
@ -366,8 +364,7 @@ SegmentNaive::QuerySlowImpl(query::QueryDeprecatedPtr query_info, Timestamp time
|
|||
auto the_offset_opt = schema_->get_offset(query_info->field_name);
|
||||
Assert(the_offset_opt.has_value());
|
||||
Assert(the_offset_opt.value() < record_.entity_vec_.size());
|
||||
auto vec_ptr =
|
||||
std::static_pointer_cast<ConcurrentVector<FloatVector>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
auto vec_ptr = std::static_pointer_cast<ConcurrentVector<float>>(record_.entity_vec_.at(the_offset_opt.value()));
|
||||
std::vector<std::priority_queue<std::pair<float, int>>> records(num_queries);
|
||||
|
||||
auto get_L2_distance = [dim](const float* a, const float* b) {
|
||||
|
|
@ -470,7 +467,7 @@ SegmentNaive::BuildVecIndexImpl(const IndexMeta::Entry& entry) {
|
|||
auto chunk_size = record_.uids_.chunk_size();
|
||||
|
||||
auto& uids = record_.uids_;
|
||||
auto entities = record_.get_entity<FloatVector>(offset);
|
||||
auto entities = record_.get_vec_entity<float>(offset);
|
||||
|
||||
std::vector<knowhere::DatasetPtr> datasets;
|
||||
for (int chunk_id = 0; chunk_id < uids.chunk_size(); ++chunk_id) {
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ SegmentSmallIndex::BuildVecIndexImpl(const IndexMeta::Entry& entry) {
|
|||
|
||||
auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(entry.type, entry.mode);
|
||||
auto& uids = record_.uids_;
|
||||
auto entities = record_.get_entity<FloatVector>(offset);
|
||||
auto entities = record_.get_vec_entity<float>(offset);
|
||||
|
||||
std::vector<knowhere::DatasetPtr> datasets;
|
||||
for (int chunk_id = 0; chunk_id < uids.chunk_size(); ++chunk_id) {
|
||||
|
|
@ -367,7 +367,7 @@ SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results
|
|||
auto key_offset = key_offset_opt.value();
|
||||
auto field_meta = schema_->operator[](key_offset);
|
||||
Assert(field_meta.get_data_type() == DataType::INT64);
|
||||
auto uids = record_.get_entity<int64_t>(key_offset);
|
||||
auto uids = record_.get_scalar_entity<int64_t>(key_offset);
|
||||
for (int64_t i = 0; i < size; ++i) {
|
||||
auto seg_offset = results.internal_seg_offsets_[i];
|
||||
auto row_id = uids->operator[](seg_offset);
|
||||
|
|
|
|||
|
|
@ -9,22 +9,10 @@
|
|||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum ErrorCode {
|
||||
Success = 0,
|
||||
UnexpectedException = 1,
|
||||
};
|
||||
|
||||
typedef struct CStatus {
|
||||
int error_code;
|
||||
const char* error_msg;
|
||||
} CStatus;
|
||||
|
||||
typedef void* CCollection;
|
||||
|
||||
CCollection
|
||||
|
|
|
|||
|
|
@ -13,52 +13,21 @@
|
|||
#include "query/Plan.h"
|
||||
#include "segcore/Collection.h"
|
||||
|
||||
CStatus
|
||||
CreatePlan(CCollection c_col, const char* dsl, CPlan* res_plan) {
|
||||
CPlan
|
||||
CreatePlan(CCollection c_col, const char* dsl) {
|
||||
auto col = (milvus::segcore::Collection*)c_col;
|
||||
auto res = milvus::query::CreatePlan(*col->get_schema(), dsl);
|
||||
|
||||
try {
|
||||
auto res = milvus::query::CreatePlan(*col->get_schema(), dsl);
|
||||
|
||||
auto status = CStatus();
|
||||
status.error_code = Success;
|
||||
status.error_msg = "";
|
||||
auto plan = (CPlan)res.release();
|
||||
*res_plan = plan;
|
||||
return status;
|
||||
} catch (std::exception& e) {
|
||||
auto status = CStatus();
|
||||
status.error_code = UnexpectedException;
|
||||
status.error_msg = strdup(e.what());
|
||||
*res_plan = nullptr;
|
||||
return status;
|
||||
}
|
||||
return (CPlan)res.release();
|
||||
}
|
||||
|
||||
CStatus
|
||||
ParsePlaceholderGroup(CPlan c_plan,
|
||||
void* placeholder_group_blob,
|
||||
int64_t blob_size,
|
||||
CPlaceholderGroup* res_placeholder_group) {
|
||||
CPlaceholderGroup
|
||||
ParsePlaceholderGroup(CPlan c_plan, void* placeholder_group_blob, int64_t blob_size) {
|
||||
std::string blob_string((char*)placeholder_group_blob, (char*)placeholder_group_blob + blob_size);
|
||||
auto plan = (milvus::query::Plan*)c_plan;
|
||||
auto res = milvus::query::ParsePlaceholderGroup(plan, blob_string);
|
||||
|
||||
try {
|
||||
auto res = milvus::query::ParsePlaceholderGroup(plan, blob_string);
|
||||
|
||||
auto status = CStatus();
|
||||
status.error_code = Success;
|
||||
status.error_msg = "";
|
||||
auto group = (CPlaceholderGroup)res.release();
|
||||
*res_placeholder_group = group;
|
||||
return status;
|
||||
} catch (std::exception& e) {
|
||||
auto status = CStatus();
|
||||
status.error_code = UnexpectedException;
|
||||
status.error_msg = strdup(e.what());
|
||||
*res_placeholder_group = nullptr;
|
||||
return status;
|
||||
}
|
||||
return (CPlaceholderGroup)res.release();
|
||||
}
|
||||
|
||||
int64_t
|
||||
|
|
|
|||
|
|
@ -20,14 +20,11 @@ extern "C" {
|
|||
typedef void* CPlan;
|
||||
typedef void* CPlaceholderGroup;
|
||||
|
||||
CStatus
|
||||
CreatePlan(CCollection col, const char* dsl, CPlan* res_plan);
|
||||
CPlan
|
||||
CreatePlan(CCollection col, const char* dsl);
|
||||
|
||||
CStatus
|
||||
ParsePlaceholderGroup(CPlan plan,
|
||||
void* placeholder_group_blob,
|
||||
int64_t blob_size,
|
||||
CPlaceholderGroup* res_placeholder_group);
|
||||
CPlaceholderGroup
|
||||
ParsePlaceholderGroup(CPlan plan, void* placeholder_group_blob, int64_t blob_size);
|
||||
|
||||
int64_t
|
||||
GetNumOfQueries(CPlaceholderGroup placeholder_group);
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ Insert(CSegmentBase c_segment,
|
|||
status.error_code = Success;
|
||||
status.error_msg = "";
|
||||
return status;
|
||||
} catch (std::exception& e) {
|
||||
} catch (std::runtime_error& e) {
|
||||
auto status = CStatus();
|
||||
status.error_code = UnexpectedException;
|
||||
status.error_msg = strdup(e.what());
|
||||
|
|
@ -103,7 +103,7 @@ Delete(
|
|||
status.error_code = Success;
|
||||
status.error_msg = "";
|
||||
return status;
|
||||
} catch (std::exception& e) {
|
||||
} catch (std::runtime_error& e) {
|
||||
auto status = CStatus();
|
||||
status.error_code = UnexpectedException;
|
||||
status.error_msg = strdup(e.what());
|
||||
|
|
@ -141,7 +141,7 @@ Search(CSegmentBase c_segment,
|
|||
auto res = segment->Search(plan, placeholder_groups.data(), timestamps, num_groups, *query_result);
|
||||
status.error_code = Success;
|
||||
status.error_msg = "";
|
||||
} catch (std::exception& e) {
|
||||
} catch (std::runtime_error& e) {
|
||||
status.error_code = UnexpectedException;
|
||||
status.error_msg = strdup(e.what());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,11 +17,22 @@ extern "C" {
|
|||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "segcore/collection_c.h"
|
||||
#include "segcore/plan_c.h"
|
||||
|
||||
typedef void* CSegmentBase;
|
||||
typedef void* CQueryResult;
|
||||
|
||||
enum ErrorCode {
|
||||
Success = 0,
|
||||
UnexpectedException = 1,
|
||||
};
|
||||
|
||||
typedef struct CStatus {
|
||||
int error_code;
|
||||
const char* error_msg;
|
||||
} CStatus;
|
||||
|
||||
CSegmentBase
|
||||
NewSegment(CCollection collection, uint64_t segment_id);
|
||||
|
||||
|
|
|
|||
|
|
@ -10,9 +10,7 @@ set(MILVUS_TEST_FILES
|
|||
test_indexing.cpp
|
||||
test_query.cpp
|
||||
test_expr.cpp
|
||||
test_bitmap.cpp
|
||||
test_binary.cpp
|
||||
)
|
||||
test_bitmap.cpp)
|
||||
add_executable(all_tests
|
||||
${MILVUS_TEST_FILES}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include "test_utils/DataGen.h"
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::query;
|
||||
using namespace milvus::segcore;
|
||||
|
||||
TEST(Binary, Insert) {
|
||||
int64_t N = 100000;
|
||||
int64_t num_queries = 10;
|
||||
int64_t topK = 5;
|
||||
auto schema = std::make_shared<Schema>();
|
||||
schema->AddField("vecbin", DataType::VECTOR_BINARY, 128);
|
||||
schema->AddField("age", DataType::INT64);
|
||||
auto dataset = DataGen(schema, N, 10);
|
||||
auto segment = CreateSegment(schema);
|
||||
segment->PreInsert(N);
|
||||
segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
|
||||
int i = 1 + 1;
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@
|
|||
#include <random>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "segcore/collection_c.h"
|
||||
#include "pb/service_msg.pb.h"
|
||||
#include "segcore/reduce_c.h"
|
||||
|
||||
|
|
@ -150,15 +151,8 @@ TEST(CApiTest, SearchTest) {
|
|||
}
|
||||
auto blob = raw_group.SerializeAsString();
|
||||
|
||||
void* plan = nullptr;
|
||||
|
||||
auto status = CreatePlan(collection, dsl_string, &plan);
|
||||
assert(status.error_code == Success);
|
||||
|
||||
void* placeholderGroup = nullptr;
|
||||
status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup);
|
||||
assert(status.error_code == Success);
|
||||
|
||||
auto plan = CreatePlan(collection, dsl_string);
|
||||
auto placeholderGroup = ParsePlaceholderGroup(plan, blob.data(), blob.length());
|
||||
std::vector<CPlaceholderGroup> placeholderGroups;
|
||||
placeholderGroups.push_back(placeholderGroup);
|
||||
timestamps.clear();
|
||||
|
|
@ -617,15 +611,8 @@ TEST(CApiTest, Reduce) {
|
|||
}
|
||||
auto blob = raw_group.SerializeAsString();
|
||||
|
||||
void* plan = nullptr;
|
||||
|
||||
auto status = CreatePlan(collection, dsl_string, &plan);
|
||||
assert(status.error_code == Success);
|
||||
|
||||
void* placeholderGroup = nullptr;
|
||||
status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup);
|
||||
assert(status.error_code == Success);
|
||||
|
||||
auto plan = CreatePlan(collection, dsl_string);
|
||||
auto placeholderGroup = ParsePlaceholderGroup(plan, blob.data(), blob.length());
|
||||
std::vector<CPlaceholderGroup> placeholderGroups;
|
||||
placeholderGroups.push_back(placeholderGroup);
|
||||
timestamps.clear();
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ TEST(ConcurrentVector, TestABI) {
|
|||
|
||||
TEST(ConcurrentVector, TestSingle) {
|
||||
auto dim = 8;
|
||||
ConcurrentVectorImpl<int, false, 32> c_vec(dim);
|
||||
ConcurrentVector<int, false, 32> c_vec(dim);
|
||||
std::default_random_engine e(42);
|
||||
int data = 0;
|
||||
auto total_count = 0;
|
||||
|
|
@ -66,7 +66,7 @@ TEST(ConcurrentVector, TestMultithreads) {
|
|||
constexpr int threads = 16;
|
||||
std::vector<int64_t> total_counts(threads);
|
||||
|
||||
ConcurrentVectorImpl<int64_t, false, 32> c_vec(dim);
|
||||
ConcurrentVector<int64_t, false, 32> c_vec(dim);
|
||||
std::atomic<int64_t> ack_counter = 0;
|
||||
// std::mutex mutex;
|
||||
|
||||
|
|
|
|||
|
|
@ -32,8 +32,6 @@
|
|||
#include <chrono>
|
||||
#include "test_utils/Timer.h"
|
||||
#include "segcore/Reduce.h"
|
||||
#include "test_utils/DataGen.h"
|
||||
#include "query/BruteForceSearch.h"
|
||||
|
||||
using std::cin;
|
||||
using std::cout;
|
||||
|
|
@ -57,7 +55,7 @@ generate_data(int N) {
|
|||
uids.push_back(10 * N + i);
|
||||
timestamps.push_back(0);
|
||||
// append vec
|
||||
vector<float> vec(DIM);
|
||||
float vec[DIM];
|
||||
for (auto& x : vec) {
|
||||
x = distribution(er);
|
||||
}
|
||||
|
|
@ -83,7 +81,6 @@ TEST(Indexing, SmartBruteForce) {
|
|||
auto [raw_data, timestamps, uids] = generate_data<DIM>(N);
|
||||
auto total_count = DIM * TOPK;
|
||||
auto raw = (const float*)raw_data.data();
|
||||
AssertInfo(raw, "wtf");
|
||||
|
||||
constexpr int64_t queries = 3;
|
||||
auto heap = faiss::float_maxheap_array_t{};
|
||||
|
|
@ -234,106 +231,3 @@ TEST(Indexing, IVFFlatNM) {
|
|||
cout << ids[i] << "->" << dis[i] << endl;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Indexing, DISABLED_BinaryBruteForce) {
|
||||
int64_t N = 100000;
|
||||
int64_t num_queries = 10;
|
||||
int64_t topk = 5;
|
||||
int64_t dim = 64;
|
||||
auto result_count = topk * num_queries;
|
||||
auto schema = std::make_shared<Schema>();
|
||||
schema->AddField("vecbin", DataType::VECTOR_BINARY, dim);
|
||||
schema->AddField("age", DataType::INT64);
|
||||
auto dataset = DataGen(schema, N, 10);
|
||||
vector<float> distances(result_count);
|
||||
vector<int64_t> ids(result_count);
|
||||
auto bin_vec = dataset.get_col<uint8_t>(0);
|
||||
auto line_sizeof = schema->operator[](0).get_sizeof();
|
||||
auto query_data = 1024 * line_sizeof + bin_vec.data();
|
||||
query::BinarySearchBruteForce(faiss::MetricType::METRIC_Jaccard, line_sizeof, bin_vec.data(), N, topk, num_queries,
|
||||
query_data, distances.data(), ids.data());
|
||||
QueryResult qr;
|
||||
qr.num_queries_ = num_queries;
|
||||
qr.topK_ = topk;
|
||||
qr.internal_seg_offsets_ = ids;
|
||||
qr.result_distances_ = distances;
|
||||
|
||||
auto json = QueryResultToJson(qr);
|
||||
auto ref = Json::parse(R"(
|
||||
[
|
||||
[
|
||||
[
|
||||
"1024->0.000000",
|
||||
"86966->0.395349",
|
||||
"24843->0.404762",
|
||||
"13806->0.416667",
|
||||
"44313->0.421053"
|
||||
],
|
||||
[
|
||||
"1025->0.000000",
|
||||
"14226->0.348837",
|
||||
"1488->0.365854",
|
||||
"47337->0.377778",
|
||||
"20913->0.377778"
|
||||
],
|
||||
[
|
||||
"1026->0.000000",
|
||||
"81882->0.386364",
|
||||
"9215->0.409091",
|
||||
"95024->0.409091",
|
||||
"54987->0.414634"
|
||||
],
|
||||
[
|
||||
"1027->0.000000",
|
||||
"68981->0.394737",
|
||||
"75528->0.404762",
|
||||
"68794->0.405405",
|
||||
"21975->0.425000"
|
||||
],
|
||||
[
|
||||
"1028->0.000000",
|
||||
"90290->0.375000",
|
||||
"34309->0.394737",
|
||||
"58559->0.400000",
|
||||
"33865->0.400000"
|
||||
],
|
||||
[
|
||||
"1029->0.000000",
|
||||
"62722->0.388889",
|
||||
"89070->0.394737",
|
||||
"18528->0.414634",
|
||||
"94971->0.421053"
|
||||
],
|
||||
[
|
||||
"1030->0.000000",
|
||||
"67402->0.333333",
|
||||
"3988->0.347826",
|
||||
"86376->0.354167",
|
||||
"84381->0.361702"
|
||||
],
|
||||
[
|
||||
"1031->0.000000",
|
||||
"81569->0.325581",
|
||||
"12715->0.347826",
|
||||
"40332->0.363636",
|
||||
"21037->0.372093"
|
||||
],
|
||||
[
|
||||
"1032->0.000000",
|
||||
"60536->0.428571",
|
||||
"93293->0.432432",
|
||||
"70969->0.435897",
|
||||
"64048->0.450000"
|
||||
],
|
||||
[
|
||||
"1033->0.000000",
|
||||
"99022->0.394737",
|
||||
"11763->0.405405",
|
||||
"50073->0.428571",
|
||||
"97118->0.428571"
|
||||
]
|
||||
]
|
||||
]
|
||||
)");
|
||||
ASSERT_EQ(json, ref);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -214,9 +214,17 @@ TEST(Query, ExecWithPredicate) {
|
|||
Timestamp time = 1000000;
|
||||
std::vector<const PlaceholderGroup*> ph_group_arr = {ph_group.get()};
|
||||
segment->Search(plan.get(), ph_group_arr.data(), &time, 1, qr);
|
||||
std::vector<std::vector<std::string>> results;
|
||||
int topk = 5;
|
||||
|
||||
Json json = QueryResultToJson(qr);
|
||||
for (int q = 0; q < num_queries; ++q) {
|
||||
std::vector<std::string> result;
|
||||
for (int k = 0; k < topk; ++k) {
|
||||
int index = q * topk + k;
|
||||
result.emplace_back(std::to_string(qr.result_ids_[index]) + "->" +
|
||||
std::to_string(qr.result_distances_[index]));
|
||||
}
|
||||
results.emplace_back(std::move(result));
|
||||
}
|
||||
|
||||
auto ref = Json::parse(R"([
|
||||
[
|
||||
|
|
@ -258,6 +266,7 @@ TEST(Query, ExecWithPredicate) {
|
|||
]
|
||||
])");
|
||||
|
||||
Json json{results};
|
||||
ASSERT_EQ(json, ref);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,24 +87,6 @@ DataGen(SchemaPtr schema, int64_t N, uint64_t seed = 42) {
|
|||
insert_cols(data);
|
||||
break;
|
||||
}
|
||||
case engine::DataType::VECTOR_BINARY: {
|
||||
auto dim = field.get_dim();
|
||||
Assert(dim % 8 == 0);
|
||||
vector<uint8_t> data(dim / 8 * N);
|
||||
for (auto& x : data) {
|
||||
x = er();
|
||||
}
|
||||
insert_cols(data);
|
||||
break;
|
||||
}
|
||||
case engine::DataType::INT64: {
|
||||
vector<int64_t> data(N);
|
||||
for (auto& x : data) {
|
||||
x = er();
|
||||
}
|
||||
insert_cols(data);
|
||||
break;
|
||||
}
|
||||
case engine::DataType::INT32: {
|
||||
vector<int> data(N);
|
||||
for (auto& x : data) {
|
||||
|
|
@ -160,41 +142,4 @@ CreatePlaceholderGroup(int64_t num_queries, int dim, int64_t seed = 42) {
|
|||
return raw_group;
|
||||
}
|
||||
|
||||
inline auto
|
||||
CreateBinaryPlaceholderGroup(int64_t num_queries, int64_t dim, int64_t seed = 42) {
|
||||
assert(dim % 8 == 0);
|
||||
namespace ser = milvus::proto::service;
|
||||
ser::PlaceholderGroup raw_group;
|
||||
auto value = raw_group.add_placeholders();
|
||||
value->set_tag("$0");
|
||||
value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
|
||||
std::default_random_engine e(seed);
|
||||
for (int i = 0; i < num_queries; ++i) {
|
||||
std::vector<uint8_t> vec;
|
||||
for (int d = 0; d < dim / 8; ++d) {
|
||||
vec.push_back(e());
|
||||
}
|
||||
// std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
|
||||
value->add_values(vec.data(), vec.size() * sizeof(float));
|
||||
}
|
||||
return raw_group;
|
||||
}
|
||||
|
||||
inline Json
|
||||
QueryResultToJson(const QueryResult& qr) {
|
||||
int64_t num_queries = qr.num_queries_;
|
||||
int64_t topk = qr.topK_;
|
||||
std::vector<std::vector<std::string>> results;
|
||||
for (int q = 0; q < num_queries; ++q) {
|
||||
std::vector<std::string> result;
|
||||
for (int k = 0; k < topk; ++k) {
|
||||
int index = q * topk + k;
|
||||
result.emplace_back(std::to_string(qr.internal_seg_offsets_[index]) + "->" +
|
||||
std::to_string(qr.result_distances_[index]));
|
||||
}
|
||||
results.emplace_back(std::move(result));
|
||||
}
|
||||
return Json{results};
|
||||
};
|
||||
|
||||
} // namespace milvus::segcore
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ func (mt *metaTable) saveCollectionMeta(coll *pb.CollectionMeta) error {
|
|||
collBytes := proto.MarshalTextString(coll)
|
||||
mt.collID2Meta[coll.ID] = *coll
|
||||
mt.collName2ID[coll.Schema.Name] = coll.ID
|
||||
return mt.client.Save("/collection/"+strconv.FormatInt(coll.ID, 10), collBytes)
|
||||
return mt.client.Save("/collection/"+strconv.FormatInt(coll.ID, 10), string(collBytes))
|
||||
}
|
||||
|
||||
// metaTable.ddLock.Lock() before call this function
|
||||
|
|
@ -119,7 +119,7 @@ func (mt *metaTable) saveSegmentMeta(seg *pb.SegmentMeta) error {
|
|||
|
||||
mt.segID2Meta[seg.SegmentID] = *seg
|
||||
|
||||
return mt.client.Save("/segment/"+strconv.FormatInt(seg.SegmentID, 10), segBytes)
|
||||
return mt.client.Save("/segment/"+strconv.FormatInt(seg.SegmentID, 10), string(segBytes))
|
||||
}
|
||||
|
||||
// metaTable.ddLock.Lock() before call this function
|
||||
|
|
@ -132,7 +132,7 @@ func (mt *metaTable) saveCollectionAndDeleteSegmentsMeta(coll *pb.CollectionMeta
|
|||
kvs := make(map[string]string)
|
||||
collStrs := proto.MarshalTextString(coll)
|
||||
|
||||
kvs["/collection/"+strconv.FormatInt(coll.ID, 10)] = collStrs
|
||||
kvs["/collection/"+strconv.FormatInt(coll.ID, 10)] = string(collStrs)
|
||||
|
||||
for _, segID := range segIDs {
|
||||
_, ok := mt.segID2Meta[segID]
|
||||
|
|
@ -152,14 +152,14 @@ func (mt *metaTable) saveCollectionsAndSegmentsMeta(coll *pb.CollectionMeta, seg
|
|||
kvs := make(map[string]string)
|
||||
collBytes := proto.MarshalTextString(coll)
|
||||
|
||||
kvs["/collection/"+strconv.FormatInt(coll.ID, 10)] = collBytes
|
||||
kvs["/collection/"+strconv.FormatInt(coll.ID, 10)] = string(collBytes)
|
||||
|
||||
mt.collID2Meta[coll.ID] = *coll
|
||||
mt.collName2ID[coll.Schema.Name] = coll.ID
|
||||
|
||||
segBytes := proto.MarshalTextString(seg)
|
||||
|
||||
kvs["/segment/"+strconv.FormatInt(seg.SegmentID, 10)] = segBytes
|
||||
kvs["/segment/"+strconv.FormatInt(seg.SegmentID, 10)] = string(segBytes)
|
||||
|
||||
mt.segID2Meta[seg.SegmentID] = *seg
|
||||
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ func TestSegmentManager_SegmentStats(t *testing.T) {
|
|||
baseMsg := msgstream.BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{1},
|
||||
HashValues: []int32{1},
|
||||
}
|
||||
msg := msgstream.QueryNodeSegStatsMsg{
|
||||
QueryNodeSegStats: stats,
|
||||
|
|
@ -358,7 +358,7 @@ func TestSegmentManager_RPC(t *testing.T) {
|
|||
},
|
||||
},
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ func (syncMsgProducer *timeSyncMsgProducer) broadcastMsg(barrier TimeTickBarrier
|
|||
baseMsg := ms.BaseMsg{
|
||||
BeginTimestamp: timetick,
|
||||
EndTimestamp: timetick,
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
}
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func (ttBarrier *softTimeTickBarrier) Start() error {
|
|||
for _, timetickmsg := range ttmsgs.Msgs {
|
||||
ttmsg := timetickmsg.(*ms.TimeTickMsg)
|
||||
oldT, ok := ttBarrier.peer2LastTt[ttmsg.PeerID]
|
||||
// log.Printf("[softTimeTickBarrier] peer(%d)=%d\n", ttmsg.PeerID, ttmsg.Timestamp)
|
||||
log.Printf("[softTimeTickBarrier] peer(%d)=%d\n", ttmsg.PeerID, ttmsg.Timestamp)
|
||||
|
||||
if !ok {
|
||||
log.Printf("[softTimeTickBarrier] Warning: peerID %d not exist\n", ttmsg.PeerID)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import (
|
|||
|
||||
func getTtMsg(msgType internalPb.MsgType, peerID UniqueID, timeStamp uint64) ms.TsMsg {
|
||||
baseMsg := ms.BaseMsg{
|
||||
HashValues: []uint32{uint32(peerID)},
|
||||
HashValues: []int32{int32(peerID)},
|
||||
}
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
|
|||
bucketValues[index] = channelID
|
||||
continue
|
||||
}
|
||||
bucketValues[index] = int32(hashValue % uint32(len(ms.producers)))
|
||||
bucketValues[index] = hashValue % int32(len(ms.producers))
|
||||
}
|
||||
reBucketValues[channelID] = bucketValues
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ func repackFunc(msgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func getTsMsg(msgType MsgType, reqID UniqueID, hashValue uint32) TsMsg {
|
||||
func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) TsMsg {
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{hashValue},
|
||||
HashValues: []int32{hashValue},
|
||||
}
|
||||
switch msgType {
|
||||
case internalPb.MsgType_kInsert:
|
||||
|
|
@ -129,11 +129,11 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue uint32) TsMsg {
|
|||
return nil
|
||||
}
|
||||
|
||||
func getTimeTickMsg(reqID UniqueID, hashValue uint32, time uint64) TsMsg {
|
||||
func getTimeTickMsg(reqID UniqueID, hashValue int32, time uint64) TsMsg {
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{hashValue},
|
||||
HashValues: []int32{hashValue},
|
||||
}
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
|
|
@ -369,7 +369,7 @@ func TestStream_PulsarMsgStream_InsertRepackFunc(t *testing.T) {
|
|||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{1, 3},
|
||||
HashValues: []int32{1, 3},
|
||||
}
|
||||
|
||||
insertRequest := internalPb.InsertRequest{
|
||||
|
|
@ -422,7 +422,7 @@ func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) {
|
|||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{1, 3},
|
||||
HashValues: []int32{1, 3},
|
||||
}
|
||||
|
||||
deleteRequest := internalPb.DeleteRequest{
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ type TsMsg interface {
|
|||
BeginTs() Timestamp
|
||||
EndTs() Timestamp
|
||||
Type() MsgType
|
||||
HashKeys() []uint32
|
||||
HashKeys() []int32
|
||||
Marshal(TsMsg) ([]byte, error)
|
||||
Unmarshal([]byte) (TsMsg, error)
|
||||
}
|
||||
|
|
@ -19,7 +19,7 @@ type TsMsg interface {
|
|||
type BaseMsg struct {
|
||||
BeginTimestamp Timestamp
|
||||
EndTimestamp Timestamp
|
||||
HashValues []uint32
|
||||
HashValues []int32
|
||||
}
|
||||
|
||||
func (bm *BaseMsg) BeginTs() Timestamp {
|
||||
|
|
@ -30,7 +30,7 @@ func (bm *BaseMsg) EndTs() Timestamp {
|
|||
return bm.EndTimestamp
|
||||
}
|
||||
|
||||
func (bm *BaseMsg) HashKeys() []uint32 {
|
||||
func (bm *BaseMsg) HashKeys() []int32 {
|
||||
return bm.HashValues
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,11 +87,11 @@ func newRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, erro
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func getInsertTask(reqID UniqueID, hashValue uint32) TsMsg {
|
||||
func getInsertTask(reqID UniqueID, hashValue int32) TsMsg {
|
||||
baseMsg := BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{hashValue},
|
||||
HashValues: []int32{hashValue},
|
||||
}
|
||||
insertRequest := internalPb.InsertRequest{
|
||||
MsgType: internalPb.MsgType_kInsert,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ message RowBatch {
|
|||
string collection_name = 1;
|
||||
string partition_tag = 2;
|
||||
repeated common.Blob row_data = 3;
|
||||
repeated uint32 hash_keys = 4;
|
||||
repeated int32 hash_keys = 4;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ type RowBatch struct {
|
|||
CollectionName string `protobuf:"bytes,1,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"`
|
||||
PartitionTag string `protobuf:"bytes,2,opt,name=partition_tag,json=partitionTag,proto3" json:"partition_tag,omitempty"`
|
||||
RowData []*commonpb.Blob `protobuf:"bytes,3,rep,name=row_data,json=rowData,proto3" json:"row_data,omitempty"`
|
||||
HashKeys []uint32 `protobuf:"varint,4,rep,packed,name=hash_keys,json=hashKeys,proto3" json:"hash_keys,omitempty"`
|
||||
HashKeys []int32 `protobuf:"varint,4,rep,packed,name=hash_keys,json=hashKeys,proto3" json:"hash_keys,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
|
|
@ -200,7 +200,7 @@ func (m *RowBatch) GetRowData() []*commonpb.Blob {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (m *RowBatch) GetHashKeys() []uint32 {
|
||||
func (m *RowBatch) GetHashKeys() []int32 {
|
||||
if m != nil {
|
||||
return m.HashKeys
|
||||
}
|
||||
|
|
@ -881,36 +881,36 @@ var fileDescriptor_b4b40b84dd2f74cb = []byte{
|
|||
0x9d, 0x41, 0x7f, 0x1a, 0xd1, 0x98, 0x18, 0x7b, 0xc4, 0x45, 0x2f, 0xa0, 0x18, 0xf1, 0x51, 0xdf,
|
||||
0x21, 0x92, 0x98, 0x7a, 0x5d, 0x6f, 0x94, 0xb6, 0x56, 0x9b, 0x73, 0x6d, 0xca, 0xba, 0xd3, 0xf1,
|
||||
0xf9, 0x00, 0xff, 0x1d, 0xf1, 0xd1, 0x1e, 0x91, 0x04, 0xad, 0xc3, 0xa2, 0x47, 0x84, 0xd7, 0xff,
|
||||
0x4c, 0xc7, 0xc2, 0xcc, 0xd7, 0xf5, 0x46, 0x19, 0x17, 0x13, 0xc3, 0x11, 0x1d, 0x0b, 0x6b, 0x04,
|
||||
0xd5, 0x53, 0x9f, 0xd8, 0xd4, 0xe3, 0xbe, 0x43, 0xa3, 0x73, 0xe2, 0xc7, 0x93, 0x9a, 0xb4, 0x49,
|
||||
0x4d, 0x68, 0x07, 0xf2, 0x72, 0x1c, 0x52, 0x95, 0x54, 0x65, 0x6b, 0xb3, 0x79, 0xdb, 0x6c, 0x9a,
|
||||
0x33, 0x71, 0x7a, 0xe3, 0x90, 0x62, 0xe5, 0x82, 0x56, 0xa0, 0x70, 0x99, 0x44, 0x15, 0x2a, 0x63,
|
||||
0x03, 0x67, 0x9a, 0xf5, 0x69, 0x0e, 0xf8, 0x6d, 0xc4, 0xe3, 0x10, 0x1d, 0x82, 0x11, 0x4e, 0x6d,
|
||||
0xc2, 0xd4, 0x54, 0x8d, 0xff, 0xff, 0x16, 0x4e, 0xa5, 0x8d, 0xe7, 0x7c, 0xad, 0x2f, 0x1a, 0x2c,
|
||||
0xbc, 0x8f, 0x69, 0x34, 0xbe, 0xfb, 0x0c, 0x36, 0xa1, 0x32, 0x37, 0x03, 0x61, 0xe6, 0xea, 0x7a,
|
||||
0x63, 0x11, 0x97, 0x67, 0x87, 0x20, 0x92, 0xf6, 0x38, 0xc2, 0x37, 0xf5, 0xb4, 0x3d, 0x8e, 0xf0,
|
||||
0xd1, 0x33, 0x58, 0x9e, 0xc1, 0xee, 0xbb, 0x49, 0x31, 0x66, 0xbe, 0xae, 0x35, 0x0c, 0x5c, 0x0d,
|
||||
0x6f, 0x14, 0x69, 0x7d, 0x84, 0xca, 0x99, 0x8c, 0x58, 0xe0, 0x62, 0x2a, 0x42, 0x1e, 0x08, 0x8a,
|
||||
0xb6, 0xa1, 0x20, 0x24, 0x91, 0xb1, 0x50, 0x79, 0x95, 0xb6, 0xd6, 0x6f, 0x1d, 0xea, 0x99, 0xfa,
|
||||
0x82, 0xb3, 0xaf, 0xa8, 0x06, 0x0b, 0xaa, 0x93, 0xd9, 0xa2, 0xa4, 0x8a, 0x75, 0x01, 0x46, 0x87,
|
||||
0x73, 0xff, 0x11, 0x43, 0x17, 0xaf, 0x43, 0x13, 0x40, 0x69, 0xde, 0xc7, 0x4c, 0xc8, 0x87, 0x01,
|
||||
0x4c, 0x77, 0x22, 0x6d, 0xf0, 0xf5, 0x4e, 0x0c, 0xe0, 0x9f, 0x83, 0x40, 0x52, 0x97, 0x46, 0x8f,
|
||||
0x8d, 0xa1, 0x4f, 0x30, 0x04, 0xd4, 0x32, 0x0c, 0x4c, 0x02, 0x97, 0x3e, 0xb8, 0x53, 0x03, 0xea,
|
||||
0xb2, 0x40, 0x75, 0x4a, 0xc7, 0xa9, 0x92, 0x2c, 0x08, 0x0d, 0x1c, 0xb5, 0x20, 0x3a, 0x4e, 0x44,
|
||||
0xeb, 0xbb, 0x06, 0xff, 0x4e, 0xb9, 0x69, 0x8f, 0x0a, 0x3b, 0x62, 0x61, 0x22, 0xde, 0x0f, 0xf6,
|
||||
0x15, 0x14, 0x52, 0xe6, 0x53, 0xb8, 0xa5, 0x9f, 0x0e, 0x32, 0x65, 0xc5, 0x29, 0xe0, 0x99, 0x32,
|
||||
0xe0, 0xcc, 0x09, 0xb5, 0x01, 0x92, 0x40, 0x4c, 0x48, 0x66, 0x8b, 0x8c, 0x48, 0xfe, 0xbb, 0x15,
|
||||
0xf7, 0x88, 0x8e, 0xd5, 0x6d, 0x9d, 0x12, 0x16, 0xe1, 0x19, 0x27, 0xeb, 0x9b, 0x06, 0xb5, 0x09,
|
||||
0x63, 0x3e, 0xb8, 0x9e, 0x97, 0x90, 0x57, 0x67, 0x99, 0x56, 0xb3, 0xf1, 0x8b, 0x7b, 0x9f, 0x25,
|
||||
0x68, 0xac, 0x1c, 0x1e, 0xa3, 0x92, 0x23, 0xc8, 0xbf, 0x63, 0x52, 0x5d, 0xf5, 0xc1, 0x5e, 0x4a,
|
||||
0x39, 0x3a, 0x4e, 0x44, 0xb4, 0x3a, 0xc3, 0xb6, 0x39, 0xc5, 0x5d, 0x13, 0x4a, 0x5d, 0x49, 0x06,
|
||||
0xc0, 0xa3, 0x8c, 0xd4, 0x72, 0x38, 0xd3, 0xac, 0x73, 0x28, 0x29, 0xce, 0xc1, 0x54, 0xc4, 0xbe,
|
||||
0xbc, 0x5f, 0x33, 0x10, 0xe4, 0x3d, 0x26, 0x45, 0x06, 0xa9, 0xe4, 0xa7, 0xaf, 0x61, 0xe9, 0x06,
|
||||
0xbb, 0xa2, 0x22, 0xe4, 0xbb, 0x27, 0xdd, 0xfd, 0xea, 0x5f, 0x68, 0x19, 0xca, 0xe7, 0xfb, 0xbb,
|
||||
0xbd, 0x13, 0xdc, 0xef, 0x1c, 0x74, 0xdb, 0xf8, 0xa2, 0xea, 0xa0, 0x2a, 0x18, 0x99, 0xe9, 0xcd,
|
||||
0xf1, 0x49, 0xbb, 0x57, 0xa5, 0x9d, 0xdd, 0x0f, 0x6d, 0x97, 0x49, 0x2f, 0x1e, 0x24, 0xa8, 0xad,
|
||||
0x2b, 0xe6, 0xfb, 0xec, 0x4a, 0x52, 0xdb, 0x6b, 0xa5, 0x19, 0x3d, 0x77, 0x98, 0x90, 0x11, 0x1b,
|
||||
0xc4, 0x92, 0x3a, 0x2d, 0x16, 0x48, 0x1a, 0x05, 0xc4, 0x6f, 0xa9, 0x34, 0x5b, 0xd9, 0x00, 0xc2,
|
||||
0xc1, 0xa0, 0xa0, 0x0c, 0xdb, 0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xee, 0x08, 0x5d, 0xa4, 0xaf,
|
||||
0x4c, 0xc7, 0xc2, 0xcc, 0xd7, 0xf5, 0xc6, 0x02, 0x2e, 0x26, 0x86, 0x23, 0x3a, 0x16, 0xd6, 0x08,
|
||||
0xaa, 0xa7, 0x3e, 0xb1, 0xa9, 0xc7, 0x7d, 0x87, 0x46, 0xe7, 0xc4, 0x8f, 0x27, 0x35, 0x69, 0x93,
|
||||
0x9a, 0xd0, 0x0e, 0xe4, 0xe5, 0x38, 0xa4, 0x2a, 0xa9, 0xca, 0xd6, 0x66, 0xf3, 0xb6, 0xd9, 0x34,
|
||||
0x67, 0xe2, 0xf4, 0xc6, 0x21, 0xc5, 0xca, 0x05, 0xad, 0x40, 0xe1, 0x32, 0x89, 0x2a, 0x54, 0xc6,
|
||||
0x06, 0xce, 0x34, 0xeb, 0xd3, 0x1c, 0xf0, 0xdb, 0x88, 0xc7, 0x21, 0x3a, 0x04, 0x23, 0x9c, 0xda,
|
||||
0x84, 0xa9, 0xa9, 0x1a, 0xff, 0xff, 0x2d, 0x9c, 0x4a, 0x1b, 0xcf, 0xf9, 0x5a, 0x5f, 0x34, 0x58,
|
||||
0x78, 0x1f, 0xd3, 0x68, 0x7c, 0xf7, 0x19, 0x6c, 0x42, 0x65, 0x6e, 0x06, 0xc2, 0xcc, 0xd5, 0xf5,
|
||||
0xc6, 0x22, 0x2e, 0xcf, 0x0e, 0x41, 0x24, 0xed, 0x71, 0x84, 0x6f, 0xea, 0x69, 0x7b, 0x1c, 0xe1,
|
||||
0xa3, 0x67, 0xb0, 0x3c, 0x83, 0xdd, 0x77, 0x93, 0x62, 0xcc, 0x7c, 0x5d, 0x6b, 0x18, 0xb8, 0x1a,
|
||||
0xde, 0x28, 0xd2, 0xfa, 0x08, 0x95, 0x33, 0x19, 0xb1, 0xc0, 0xc5, 0x54, 0x84, 0x3c, 0x10, 0x14,
|
||||
0x6d, 0x43, 0x41, 0x48, 0x22, 0x63, 0xa1, 0xf2, 0x2a, 0x6d, 0xad, 0xdf, 0x3a, 0xd4, 0x33, 0xf5,
|
||||
0x05, 0x67, 0x5f, 0x51, 0x0d, 0x16, 0x54, 0x27, 0xb3, 0x45, 0x49, 0x15, 0xeb, 0x02, 0x8c, 0x0e,
|
||||
0xe7, 0xfe, 0x23, 0x86, 0x2e, 0x5e, 0x87, 0x26, 0x80, 0xd2, 0xbc, 0x8f, 0x99, 0x90, 0x0f, 0x03,
|
||||
0x98, 0xee, 0x44, 0xda, 0xe0, 0xeb, 0x9d, 0x18, 0xc0, 0x3f, 0x07, 0x81, 0xa4, 0x2e, 0x8d, 0x1e,
|
||||
0x1b, 0x43, 0x9f, 0x60, 0x08, 0xa8, 0x65, 0x18, 0x98, 0x04, 0x2e, 0x7d, 0x70, 0xa7, 0x06, 0xd4,
|
||||
0x65, 0x81, 0xea, 0x94, 0x8e, 0x53, 0x25, 0x59, 0x10, 0x1a, 0x38, 0x6a, 0x41, 0x74, 0x9c, 0x88,
|
||||
0xd6, 0x77, 0x0d, 0xfe, 0x9d, 0x72, 0xd3, 0x1e, 0x15, 0x76, 0xc4, 0xc2, 0x44, 0xbc, 0x1f, 0xec,
|
||||
0x2b, 0x28, 0xa4, 0xcc, 0xa7, 0x70, 0x4b, 0x3f, 0x1d, 0x64, 0xca, 0x8a, 0x53, 0xc0, 0x33, 0x65,
|
||||
0xc0, 0x99, 0x13, 0x6a, 0x03, 0x24, 0x81, 0x98, 0x90, 0xcc, 0x16, 0x19, 0x91, 0xfc, 0x77, 0x2b,
|
||||
0xee, 0x11, 0x1d, 0xab, 0xdb, 0x3a, 0x25, 0x2c, 0xc2, 0x33, 0x4e, 0xd6, 0x37, 0x0d, 0x6a, 0x13,
|
||||
0xc6, 0x7c, 0x70, 0x3d, 0x2f, 0x21, 0xaf, 0xce, 0x32, 0xad, 0x66, 0xe3, 0x17, 0xf7, 0x3e, 0x4b,
|
||||
0xd0, 0x58, 0x39, 0x3c, 0x46, 0x25, 0x47, 0x90, 0x7f, 0xc7, 0xa4, 0xba, 0xea, 0x83, 0xbd, 0x94,
|
||||
0x72, 0x74, 0x9c, 0x88, 0x68, 0x75, 0x86, 0x6d, 0x73, 0x8a, 0xbb, 0x26, 0x94, 0xba, 0x92, 0x0c,
|
||||
0x80, 0x47, 0x19, 0xa9, 0xe5, 0x70, 0xa6, 0x59, 0xe7, 0x50, 0x52, 0x9c, 0x83, 0xa9, 0x88, 0x7d,
|
||||
0x79, 0xbf, 0x66, 0x20, 0xc8, 0x7b, 0x4c, 0x8a, 0x0c, 0x52, 0xc9, 0x4f, 0x5f, 0xc3, 0xd2, 0x0d,
|
||||
0x76, 0x45, 0x45, 0xc8, 0x77, 0x4f, 0xba, 0xfb, 0xd5, 0xbf, 0xd0, 0x32, 0x94, 0xcf, 0xf7, 0x77,
|
||||
0x7b, 0x27, 0xb8, 0xdf, 0x39, 0xe8, 0xb6, 0xf1, 0x45, 0xd5, 0x41, 0x55, 0x30, 0x32, 0xd3, 0x9b,
|
||||
0xe3, 0x93, 0x76, 0xaf, 0x4a, 0x3b, 0xbb, 0x1f, 0xda, 0x2e, 0x93, 0x5e, 0x3c, 0x48, 0x50, 0x5b,
|
||||
0x57, 0xcc, 0xf7, 0xd9, 0x95, 0xa4, 0xb6, 0xd7, 0x4a, 0x33, 0x7a, 0xee, 0x30, 0x21, 0x23, 0x36,
|
||||
0x88, 0x25, 0x75, 0x5a, 0x2c, 0x90, 0x34, 0x0a, 0x88, 0xdf, 0x52, 0x69, 0xb6, 0xb2, 0x01, 0x84,
|
||||
0x83, 0x41, 0x41, 0x19, 0xb6, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0x33, 0xc8, 0x08, 0xe2, 0xaf,
|
||||
0x07, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ import (
|
|||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
)
|
||||
|
||||
type Cache interface {
|
||||
type MetaCache interface {
|
||||
Hit(collectionName string) bool
|
||||
Get(collectionName string) (*servicepb.CollectionDescription, error)
|
||||
Update(collectionName string) error
|
||||
Remove(collectionName string) error
|
||||
//Write(collectionName string, schema *servicepb.CollectionDescription) error
|
||||
}
|
||||
|
||||
var globalMetaCache Cache
|
||||
var globalMetaCache MetaCache
|
||||
|
||||
type SimpleMetaCache struct {
|
||||
mu sync.RWMutex
|
||||
|
|
@ -30,54 +30,29 @@ type SimpleMetaCache struct {
|
|||
ctx context.Context
|
||||
}
|
||||
|
||||
func (metaCache *SimpleMetaCache) Hit(collectionName string) bool {
|
||||
metaCache.mu.RLock()
|
||||
defer metaCache.mu.RUnlock()
|
||||
_, ok := metaCache.metas[collectionName]
|
||||
func (smc *SimpleMetaCache) Hit(collectionName string) bool {
|
||||
smc.mu.RLock()
|
||||
defer smc.mu.RUnlock()
|
||||
_, ok := smc.metas[collectionName]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (metaCache *SimpleMetaCache) Get(collectionName string) (*servicepb.CollectionDescription, error) {
|
||||
metaCache.mu.RLock()
|
||||
defer metaCache.mu.RUnlock()
|
||||
schema, ok := metaCache.metas[collectionName]
|
||||
func (smc *SimpleMetaCache) Get(collectionName string) (*servicepb.CollectionDescription, error) {
|
||||
smc.mu.RLock()
|
||||
defer smc.mu.RUnlock()
|
||||
schema, ok := smc.metas[collectionName]
|
||||
if !ok {
|
||||
return nil, errors.New("collection meta miss")
|
||||
}
|
||||
return schema, nil
|
||||
}
|
||||
|
||||
func (metaCache *SimpleMetaCache) Update(collectionName string) error {
|
||||
reqID, err := metaCache.reqIDAllocator.AllocOne()
|
||||
func (smc *SimpleMetaCache) Update(collectionName string) error {
|
||||
reqID, err := smc.reqIDAllocator.AllocOne()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ts, err := metaCache.tsoAllocator.AllocOne()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hasCollectionReq := &internalpb.HasCollectionRequest{
|
||||
MsgType: internalpb.MsgType_kHasCollection,
|
||||
ReqID: reqID,
|
||||
Timestamp: ts,
|
||||
ProxyID: metaCache.proxyID,
|
||||
CollectionName: &servicepb.CollectionName{
|
||||
CollectionName: collectionName,
|
||||
},
|
||||
}
|
||||
has, err := metaCache.masterClient.HasCollection(metaCache.ctx, hasCollectionReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has.Value {
|
||||
return errors.New("collection " + collectionName + " not exists")
|
||||
}
|
||||
|
||||
reqID, err = metaCache.reqIDAllocator.AllocOne()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ts, err = metaCache.tsoAllocator.AllocOne()
|
||||
ts, err := smc.tsoAllocator.AllocOne()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -85,32 +60,20 @@ func (metaCache *SimpleMetaCache) Update(collectionName string) error {
|
|||
MsgType: internalpb.MsgType_kDescribeCollection,
|
||||
ReqID: reqID,
|
||||
Timestamp: ts,
|
||||
ProxyID: metaCache.proxyID,
|
||||
ProxyID: smc.proxyID,
|
||||
CollectionName: &servicepb.CollectionName{
|
||||
CollectionName: collectionName,
|
||||
},
|
||||
}
|
||||
resp, err := metaCache.masterClient.DescribeCollection(metaCache.ctx, req)
|
||||
|
||||
resp, err := smc.masterClient.DescribeCollection(smc.ctx, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metaCache.mu.Lock()
|
||||
defer metaCache.mu.Unlock()
|
||||
metaCache.metas[collectionName] = resp
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (metaCache *SimpleMetaCache) Remove(collectionName string) error {
|
||||
metaCache.mu.Lock()
|
||||
defer metaCache.mu.Unlock()
|
||||
|
||||
_, ok := metaCache.metas[collectionName]
|
||||
if !ok {
|
||||
return errors.New("cannot find collection: " + collectionName)
|
||||
}
|
||||
delete(metaCache.metas, collectionName)
|
||||
smc.mu.Lock()
|
||||
defer smc.mu.Unlock()
|
||||
smc.metas[collectionName] = resp
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ func TestProxy_Insert(t *testing.T) {
|
|||
CollectionName: collectionName,
|
||||
PartitionTag: "haha",
|
||||
RowData: make([]*commonpb.Blob, 0),
|
||||
HashKeys: make([]uint32, 0),
|
||||
HashKeys: make([]int32, 0),
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
|
|
@ -281,7 +281,7 @@ func TestProxy_Search(t *testing.T) {
|
|||
for j := 0; j < 4; j++ {
|
||||
searchResultMsg := &msgstream.SearchResultMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{1},
|
||||
HashValues: []int32{1},
|
||||
},
|
||||
SearchResult: internalpb.SearchResult{
|
||||
MsgType: internalpb.MsgType_kSearchResult,
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ import (
|
|||
"math"
|
||||
"strconv"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
|
|
@ -101,20 +99,13 @@ func (it *InsertTask) Execute() error {
|
|||
return err
|
||||
}
|
||||
autoID := description.Schema.AutoID
|
||||
var rowIDBegin UniqueID
|
||||
var rowIDEnd UniqueID
|
||||
if autoID || true {
|
||||
if it.HashValues == nil || len(it.HashValues) == 0 {
|
||||
it.HashValues = make([]uint32, 0)
|
||||
}
|
||||
rowNums := len(it.BaseInsertTask.RowData)
|
||||
rowIDBegin, rowIDEnd, _ = it.rowIDAllocator.Alloc(uint32(rowNums))
|
||||
rowIDBegin, rowIDEnd, _ := it.rowIDAllocator.Alloc(uint32(rowNums))
|
||||
it.BaseInsertTask.RowIDs = make([]UniqueID, rowNums)
|
||||
for i := rowIDBegin; i < rowIDEnd; i++ {
|
||||
offset := i - rowIDBegin
|
||||
it.BaseInsertTask.RowIDs[offset] = i
|
||||
hashValue, _ := typeutil.Hash32Int64(i)
|
||||
it.HashValues = append(it.HashValues, hashValue)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,8 +121,6 @@ func (it *InsertTask) Execute() error {
|
|||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_SUCCESS,
|
||||
},
|
||||
Begin: rowIDBegin,
|
||||
End: rowIDEnd,
|
||||
}
|
||||
if err != nil {
|
||||
it.result.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
|
||||
|
|
@ -298,7 +287,7 @@ func (dct *DropCollectionTask) Execute() error {
|
|||
}
|
||||
|
||||
func (dct *DropCollectionTask) PostExecute() error {
|
||||
return globalMetaCache.Remove(dct.CollectionName.CollectionName)
|
||||
return nil
|
||||
}
|
||||
|
||||
type QueryTask struct {
|
||||
|
|
@ -336,18 +325,6 @@ func (qt *QueryTask) SetTs(ts Timestamp) {
|
|||
}
|
||||
|
||||
func (qt *QueryTask) PreExecute() error {
|
||||
collectionName := qt.query.CollectionName
|
||||
if !globalMetaCache.Hit(collectionName) {
|
||||
err := globalMetaCache.Update(collectionName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := globalMetaCache.Get(collectionName)
|
||||
if err != nil { // err is not nil if collection not exists
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ValidateCollectionName(qt.query.CollectionName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -375,7 +352,7 @@ func (qt *QueryTask) Execute() error {
|
|||
var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{
|
||||
SearchRequest: qt.SearchRequest,
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{uint32(Params.ProxyID())},
|
||||
HashValues: []int32{int32(Params.ProxyID())},
|
||||
BeginTimestamp: qt.Timestamp,
|
||||
EndTimestamp: qt.Timestamp,
|
||||
},
|
||||
|
|
@ -401,33 +378,25 @@ func (qt *QueryTask) PostExecute() error {
|
|||
log.Print("wait to finish failed, timeout!")
|
||||
return errors.New("wait to finish failed, timeout")
|
||||
case searchResults := <-qt.resultBuf:
|
||||
filterSearchResult := make([]*internalpb.SearchResult, 0)
|
||||
for _, partialSearchResult := range searchResults {
|
||||
if partialSearchResult.Status.ErrorCode == commonpb.ErrorCode_SUCCESS {
|
||||
filterSearchResult = append(filterSearchResult, partialSearchResult)
|
||||
}
|
||||
}
|
||||
|
||||
rlen := len(filterSearchResult) // query num
|
||||
rlen := len(searchResults) // query num
|
||||
if rlen <= 0 {
|
||||
qt.result = &servicepb.QueryResult{}
|
||||
return nil
|
||||
}
|
||||
|
||||
n := len(filterSearchResult[0].Hits) // n
|
||||
n := len(searchResults[0].Hits) // n
|
||||
if n <= 0 {
|
||||
qt.result = &servicepb.QueryResult{}
|
||||
return nil
|
||||
}
|
||||
|
||||
hits := make([][]*servicepb.Hits, rlen)
|
||||
for i, partialSearchResult := range filterSearchResult {
|
||||
for i, searchResult := range searchResults {
|
||||
hits[i] = make([]*servicepb.Hits, n)
|
||||
for j, bs := range partialSearchResult.Hits {
|
||||
for j, bs := range searchResult.Hits {
|
||||
hits[i][j] = &servicepb.Hits{}
|
||||
err := proto.Unmarshal(bs, hits[i][j])
|
||||
if err != nil {
|
||||
log.Println("unmarshal error")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -459,17 +428,6 @@ func (qt *QueryTask) PostExecute() error {
|
|||
}
|
||||
}
|
||||
choiceOffset := locs[choice]
|
||||
// check if distance is valid, `invalid` here means very very big,
|
||||
// in this process, distance here is the smallest, so the rest of distance are all invalid
|
||||
if hits[choice][i].Scores[choiceOffset] >= float32(math.MaxFloat32) {
|
||||
qt.result = &servicepb.QueryResult{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "topk in dsl greater than the row nums of collection",
|
||||
},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
reducedHits.IDs = append(reducedHits.IDs, hits[choice][i].IDs[choiceOffset])
|
||||
if hits[choice][i].RowData != nil && len(hits[choice][i].RowData) > 0 {
|
||||
reducedHits.RowData = append(reducedHits.RowData, hits[choice][i].RowData[choiceOffset])
|
||||
|
|
@ -479,7 +437,6 @@ func (qt *QueryTask) PostExecute() error {
|
|||
}
|
||||
reducedHitsBs, err := proto.Marshal(reducedHits)
|
||||
if err != nil {
|
||||
log.Println("marshal error")
|
||||
return err
|
||||
}
|
||||
qt.result.Hits = append(qt.result.Hits, reducedHitsBs)
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ func (tt *timeTick) tick() error {
|
|||
msgPack := msgstream.MsgPack{}
|
||||
timeTickMsg := &msgstream.TimeTickMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{uint32(Params.ProxyID())},
|
||||
HashValues: []int32{int32(Params.ProxyID())},
|
||||
},
|
||||
TimeTickMsg: internalpb.TimeTickMsg{
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
|
|
|
|||
|
|
@ -129,8 +129,8 @@ func TestDataSyncService_Start(t *testing.T) {
|
|||
for i := 0; i < msgLength; i++ {
|
||||
var msg msgstream.TsMsg = &msgstream.InsertMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{
|
||||
uint32(i), uint32(i),
|
||||
HashValues: []int32{
|
||||
int32(i), int32(i),
|
||||
},
|
||||
},
|
||||
InsertRequest: internalPb.InsertRequest{
|
||||
|
|
@ -163,7 +163,7 @@ func TestDataSyncService_Start(t *testing.T) {
|
|||
baseMsg := msgstream.BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
}
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ package querynode
|
|||
*/
|
||||
import "C"
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
|
|
@ -19,21 +17,11 @@ type Plan struct {
|
|||
cPlan C.CPlan
|
||||
}
|
||||
|
||||
func createPlan(col Collection, dsl string) (*Plan, error) {
|
||||
func createPlan(col Collection, dsl string) *Plan {
|
||||
cDsl := C.CString(dsl)
|
||||
var cPlan C.CPlan
|
||||
status := C.CreatePlan(col.collectionPtr, cDsl, &cPlan)
|
||||
|
||||
errorCode := status.error_code
|
||||
|
||||
if errorCode != 0 {
|
||||
errorMsg := C.GoString(status.error_msg)
|
||||
defer C.free(unsafe.Pointer(status.error_msg))
|
||||
return nil, errors.New("Insert failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
|
||||
}
|
||||
|
||||
cPlan := C.CreatePlan(col.collectionPtr, cDsl)
|
||||
var newPlan = &Plan{cPlan: cPlan}
|
||||
return newPlan, nil
|
||||
return newPlan
|
||||
}
|
||||
|
||||
func (plan *Plan) getTopK() int64 {
|
||||
|
|
@ -49,22 +37,12 @@ type PlaceholderGroup struct {
|
|||
cPlaceholderGroup C.CPlaceholderGroup
|
||||
}
|
||||
|
||||
func parserPlaceholderGroup(plan *Plan, placeHolderBlob []byte) (*PlaceholderGroup, error) {
|
||||
func parserPlaceholderGroup(plan *Plan, placeHolderBlob []byte) *PlaceholderGroup {
|
||||
var blobPtr = unsafe.Pointer(&placeHolderBlob[0])
|
||||
blobSize := C.long(len(placeHolderBlob))
|
||||
var cPlaceholderGroup C.CPlaceholderGroup
|
||||
status := C.ParsePlaceholderGroup(plan.cPlan, blobPtr, blobSize, &cPlaceholderGroup)
|
||||
|
||||
errorCode := status.error_code
|
||||
|
||||
if errorCode != 0 {
|
||||
errorMsg := C.GoString(status.error_msg)
|
||||
defer C.free(unsafe.Pointer(status.error_msg))
|
||||
return nil, errors.New("Insert failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
|
||||
}
|
||||
|
||||
cPlaceholderGroup := C.ParsePlaceholderGroup(plan.cPlan, blobPtr, blobSize)
|
||||
var newPlaceholderGroup = &PlaceholderGroup{cPlaceholderGroup: cPlaceholderGroup}
|
||||
return newPlaceholderGroup, nil
|
||||
return newPlaceholderGroup
|
||||
}
|
||||
|
||||
func (pg *PlaceholderGroup) getNumOfQuery() int64 {
|
||||
|
|
|
|||
|
|
@ -64,8 +64,7 @@ func TestPlan_Plan(t *testing.T) {
|
|||
|
||||
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
|
||||
|
||||
plan, err := createPlan(*collection, dslString)
|
||||
assert.NoError(t, err)
|
||||
plan := createPlan(*collection, dslString)
|
||||
assert.NotEqual(t, plan, nil)
|
||||
topk := plan.getTopK()
|
||||
assert.Equal(t, int(topk), 10)
|
||||
|
|
@ -123,8 +122,7 @@ func TestPlan_PlaceholderGroup(t *testing.T) {
|
|||
|
||||
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
|
||||
|
||||
plan, err := createPlan(*collection, dslString)
|
||||
assert.NoError(t, err)
|
||||
plan := createPlan(*collection, dslString)
|
||||
assert.NotNil(t, plan)
|
||||
|
||||
var searchRawData1 []byte
|
||||
|
|
@ -153,8 +151,7 @@ func TestPlan_PlaceholderGroup(t *testing.T) {
|
|||
|
||||
placeGroupByte, err := proto.Marshal(&placeholderGroup)
|
||||
assert.Nil(t, err)
|
||||
holder, err := parserPlaceholderGroup(plan, placeGroupByte)
|
||||
assert.NoError(t, err)
|
||||
holder := parserPlaceholderGroup(plan, placeGroupByte)
|
||||
assert.NotNil(t, holder)
|
||||
numQueries := holder.getNumOfQuery()
|
||||
assert.Equal(t, int(numQueries), 2)
|
||||
|
|
|
|||
|
|
@ -99,10 +99,8 @@ func TestReduce_AllFunc(t *testing.T) {
|
|||
log.Print("marshal placeholderGroup failed")
|
||||
}
|
||||
|
||||
plan, err := createPlan(*collection, dslString)
|
||||
assert.NoError(t, err)
|
||||
holder, err := parserPlaceholderGroup(plan, placeGroupByte)
|
||||
assert.NoError(t, err)
|
||||
plan := createPlan(*collection, dslString)
|
||||
holder := parserPlaceholderGroup(plan, placeGroupByte)
|
||||
placeholderGroups := make([]*PlaceholderGroup, 0)
|
||||
placeholderGroups = append(placeholderGroups, holder)
|
||||
|
||||
|
|
|
|||
|
|
@ -139,10 +139,10 @@ func (ss *searchService) receiveSearchMsg() {
|
|||
err := ss.search(msg)
|
||||
if err != nil {
|
||||
log.Println("search Failed, error msg type: ", msg.Type())
|
||||
err = ss.publishFailedSearchResult(msg)
|
||||
if err != nil {
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err)
|
||||
}
|
||||
}
|
||||
err = ss.publishFailedSearchResult(msg)
|
||||
if err != nil {
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err)
|
||||
}
|
||||
}
|
||||
log.Println("ReceiveSearchMsg, do search done, num of searchMsg = ", len(searchMsg))
|
||||
|
|
@ -191,10 +191,10 @@ func (ss *searchService) doUnsolvedMsgSearch() {
|
|||
err := ss.search(msg)
|
||||
if err != nil {
|
||||
log.Println("search Failed, error msg type: ", msg.Type())
|
||||
err = ss.publishFailedSearchResult(msg)
|
||||
if err != nil {
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err)
|
||||
}
|
||||
}
|
||||
err = ss.publishFailedSearchResult(msg)
|
||||
if err != nil {
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err)
|
||||
}
|
||||
}
|
||||
log.Println("doUnsolvedMsgSearch, do search done, num of searchMsg = ", len(searchMsg))
|
||||
|
|
@ -225,15 +225,9 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
|||
}
|
||||
collectionID := collection.ID()
|
||||
dsl := query.Dsl
|
||||
plan, err := createPlan(*collection, dsl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plan := createPlan(*collection, dsl)
|
||||
placeHolderGroupBlob := query.PlaceholderGroup
|
||||
placeholderGroup, err := parserPlaceholderGroup(plan, placeHolderGroupBlob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
placeholderGroup := parserPlaceholderGroup(plan, placeHolderGroupBlob)
|
||||
placeholderGroups := make([]*PlaceholderGroup, 0)
|
||||
placeholderGroups = append(placeholderGroups, placeholderGroup)
|
||||
|
||||
|
|
@ -257,7 +251,12 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
|||
}
|
||||
|
||||
if len(searchResults) <= 0 {
|
||||
return errors.New("search Failed, invalid partitionTag")
|
||||
log.Println("search Failed, invalid partitionTag")
|
||||
err = ss.publishFailedSearchResult(msg)
|
||||
if err != nil {
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
reducedSearchResult := reduceSearchResults(searchResults, int64(len(searchResults)))
|
||||
|
|
@ -297,7 +296,7 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
|||
Hits: hits,
|
||||
}
|
||||
searchResultMsg := &msgstream.SearchResultMsg{
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []int32{0}},
|
||||
SearchResult: results,
|
||||
}
|
||||
err = ss.publishSearchResult(searchResultMsg)
|
||||
|
|
@ -342,7 +341,7 @@ func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg) error {
|
|||
}
|
||||
|
||||
tsMsg := &msgstream.SearchResultMsg{
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []int32{0}},
|
||||
SearchResult: results,
|
||||
}
|
||||
msgPack.Msgs = append(msgPack.Msgs, tsMsg)
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ func TestSearch_Search(t *testing.T) {
|
|||
|
||||
searchMsg := &msgstream.SearchMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
},
|
||||
SearchRequest: internalpb.SearchRequest{
|
||||
MsgType: internalpb.MsgType_kSearch,
|
||||
|
|
@ -188,8 +188,8 @@ func TestSearch_Search(t *testing.T) {
|
|||
|
||||
var msg msgstream.TsMsg = &msgstream.InsertMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{
|
||||
uint32(i),
|
||||
HashValues: []int32{
|
||||
int32(i),
|
||||
},
|
||||
},
|
||||
InsertRequest: internalpb.InsertRequest{
|
||||
|
|
@ -221,7 +221,7 @@ func TestSearch_Search(t *testing.T) {
|
|||
baseMsg := msgstream.BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
}
|
||||
timeTickResult := internalpb.TimeTickMsg{
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
|
|
|
|||
|
|
@ -690,10 +690,8 @@ func TestSegment_segmentSearch(t *testing.T) {
|
|||
}
|
||||
|
||||
searchTimestamp := Timestamp(1020)
|
||||
plan, err := createPlan(*collection, dslString)
|
||||
assert.NoError(t, err)
|
||||
holder, err := parserPlaceholderGroup(plan, placeHolderGroupBlob)
|
||||
assert.NoError(t, err)
|
||||
plan := createPlan(*collection, dslString)
|
||||
holder := parserPlaceholderGroup(plan, placeHolderGroupBlob)
|
||||
placeholderGroups := make([]*PlaceholderGroup, 0)
|
||||
placeholderGroups = append(placeholderGroups, holder)
|
||||
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ func (sService *statsService) sendSegmentStatistic() {
|
|||
func (sService *statsService) publicStatistic(statistic *internalpb.QueryNodeSegStats) {
|
||||
var msg msgstream.TsMsg = &msgstream.QueryNodeSegStatsMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{0},
|
||||
HashValues: []int32{0},
|
||||
},
|
||||
QueryNodeSegStats: *statistic,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,17 +82,8 @@ func (gp *BaseTable) LoadRange(key, endKey string, limit int) ([]string, []strin
|
|||
func (gp *BaseTable) LoadYaml(fileName string) error {
|
||||
config := viper.New()
|
||||
_, fpath, _, _ := runtime.Caller(0)
|
||||
configFile := path.Dir(fpath) + "/../../../configs/" + fileName
|
||||
_, err := os.Stat(configFile)
|
||||
if os.IsNotExist(err) {
|
||||
runPath, err := os.Getwd()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
configFile = runPath + "/configs/" + fileName
|
||||
}
|
||||
|
||||
config.SetConfigFile(configFile)
|
||||
configPath := path.Dir(fpath) + "/../../../configs/"
|
||||
config.SetConfigFile(configPath + fileName)
|
||||
if err := config.ReadInConfig(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue