milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unnecessary memcpy (#2337) 

* fix #2264

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* typo

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* fix ut

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* typo

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* Remove unnecessary memcpy

Signed-off-by: yhmo <yihua.mo@zilliz.com>
pull/2341/head^2
groot 2020-05-14 05:00:23 -05:00 committed by GitHub
parent 312df5af31
commit 244d0e3405
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 37 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
core/src/db/insert/VectorSource.cpp
View File

@ -18,6 +18,7 @@
#include "db/engine/ExecutionEngine.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace engine {
@ -36,9 +37,8 @@ VectorSource::VectorSource(milvus::engine::VectorsData vectors,
}
Status
VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add,
size_t& num_vectors_added) {
VectorSource::Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added) {
uint64_t n = vectors_.vector_count_;
server::CollectAddMetrics metrics(n, table_file_schema.dimension_);
@ -61,35 +61,17 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment:
Status status;
if (!vectors_.float_data_.empty()) {
/*
status = execution_engine->AddWithIds(
num_vectors_added, vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,
vector_ids_to_add.data());
*/
std::vector<uint8_t> vectors;
LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert float data into segment";
auto size = num_vectors_added * table_file_schema.dimension_ * sizeof(float);
vectors.resize(size);
memcpy(vectors.data(), vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,
size);
LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";
status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);
float* ptr = vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_;
status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, (uint8_t*)ptr, size, vector_ids_to_add);
} else if (!vectors_.binary_data_.empty()) {
/*
status = execution_engine->AddWithIds(
num_vectors_added,
vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),
vector_ids_to_add.data());
*/
LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert binary data into segment";
std::vector<uint8_t> vectors;
auto size = num_vectors_added * SingleVectorSize(table_file_schema.dimension_) * sizeof(uint8_t);
vectors.resize(size);
memcpy(
vectors.data(),
vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),
size);
LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";
status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);
uint8_t* ptr =
vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_);
status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, ptr, size, vector_ids_to_add);
}
// Clear vector data
@ -175,6 +157,7 @@ VectorSource::SingleVectorSize(uint16_t dimension) {
return 0;
}
size_t
VectorSource::SingleEntitySize(uint16_t dimension) {
// TODO(yukun) add entity type and size compute

4
core/src/db/insert/VectorSource.h
View File

@ -36,8 +36,8 @@ class VectorSource {
const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data);
Status
Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,
const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added);
Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added);
Status
AddEntities(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& collection_file_schema,

2
core/src/db/merge/MergeLayeredStrategy.cpp
View File

@ -20,7 +20,7 @@
namespace milvus {
namespace engine {
const int64_t FORCE_MERGE_THREASHOLD = 10; // force merge files older this time(in second)
const int64_t FORCE_MERGE_THREASHOLD = 30; // force merge files older this time(in second)
Status
MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) {

10
core/src/segment/SegmentWriter.cpp
View File

@ -52,6 +52,16 @@ SegmentWriter::AddVectors(const std::string& name, const std::vector<uint8_t>& d
return Status::OK();
}
Status
SegmentWriter::AddVectors(const std::string& name, const uint8_t* data, uint64_t size,
const std::vector<doc_id_t>& uids) {
segment_ptr_->vectors_ptr_->AddData(data, size);
segment_ptr_->vectors_ptr_->AddUids(uids);
segment_ptr_->vectors_ptr_->SetName(name);
return Status::OK();
}
Status
SegmentWriter::AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,
const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data,

3
core/src/segment/SegmentWriter.h
View File

@ -36,6 +36,9 @@ class SegmentWriter {
Status
AddVectors(const std::string& name, const std::vector<uint8_t>& data, const std::vector<doc_id_t>& uids);
Status
AddVectors(const std::string& name, const uint8_t* data, uint64_t size, const std::vector<doc_id_t>& uids);
Status
AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,
const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data, const std::vector<doc_id_t>& uids);

7
core/src/segment/Vectors.cpp
View File

@ -34,6 +34,13 @@ Vectors::AddData(const std::vector<uint8_t>& data) {
data_.insert(data_.end(), std::make_move_iterator(data.begin()), std::make_move_iterator(data.end()));
}
void
Vectors::AddData(const uint8_t* data, uint64_t size) {
int64_t old_size = data_.size();
data_.resize(data_.size() + size);
memcpy(data_.data() + old_size, data, size);
}
void
Vectors::AddUids(const std::vector<doc_id_t>& uids) {
uids_.reserve(uids_.size() + uids.size());

3
core/src/segment/Vectors.h
View File

@ -33,6 +33,9 @@ class Vectors {
void
AddData(const std::vector<uint8_t>& data);
void
AddData(const uint8_t* data, uint64_t size);
void
AddUids(const std::vector<doc_id_t>& uids);