milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'main/master' into caiyd_1547_rename_storage_file

pull/1582/head
yudong.cai 2020-03-11 15:31:04 +08:00
parent 98da92afd2 40c7e8baa2
commit 539ce69305
2 changed files with 31 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -40,7 +40,7 @@ Please mark all change in change log and use the issue from GitHub
- \#1507 set_config for insert_buffer_size is wrong - \#1507 set_config for insert_buffer_size is wrong
- \#1510 Add set interfaces for WAL configurations - \#1510 Add set interfaces for WAL configurations
- \#1511 Fix big integer cannot pass to server correctly - \#1511 Fix big integer cannot pass to server correctly
- \#1517 result is not correct when search vectors in multi partition, index type is RNSG - \#1517 Result is not correct when search vectors in multi partition, index type is RNSG
- \#1518 Table count did not match after deleting vectors and compact - \#1518 Table count did not match after deleting vectors and compact
- \#1521 Make cache_insert_data take effect in-service - \#1521 Make cache_insert_data take effect in-service
- \#1525 Add setter API for config preload_table - \#1525 Add setter API for config preload_table

51
core/src/db/insert/MemTable.cpp
View File

@ -179,7 +179,7 @@ MemTable::ApplyDeletes() {
auto start_total = std::chrono::high_resolution_clock::now(); auto start_total = std::chrono::high_resolution_clock::now();
auto start = std::chrono::high_resolution_clock::now(); // auto start = std::chrono::high_resolution_clock::now();
std::vector<int> file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX, std::vector<int> file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX,
meta::TableFileSchema::FILE_TYPE::BACKUP}; meta::TableFileSchema::FILE_TYPE::BACKUP};
@ -220,9 +220,9 @@ MemTable::ApplyDeletes() {
OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_check); OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_check);
auto end = std::chrono::high_resolution_clock::now(); auto time0 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = end - start; std::chrono::duration<double> diff0 = time0 - start_total;
ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff.count() << " s"; ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff0.count() << " s";
meta::TableFilesSchema table_files_to_update; meta::TableFilesSchema table_files_to_update;
@ -230,7 +230,7 @@ MemTable::ApplyDeletes() {
auto& table_file = table_files[kv.first]; auto& table_file = table_files[kv.first];
ENGINE_LOG_DEBUG << "Applying deletes in segment: " << table_file.segment_id_; ENGINE_LOG_DEBUG << "Applying deletes in segment: " << table_file.segment_id_;
start = std::chrono::high_resolution_clock::now(); auto time1 = std::chrono::high_resolution_clock::now();
std::string segment_dir; std::string segment_dir;
utils::GetParentPath(table_file.location_, segment_dir); utils::GetParentPath(table_file.location_, segment_dir);
@ -274,17 +274,15 @@ MemTable::ApplyDeletes() {
segment::DeletedDocsPtr deleted_docs = std::make_shared<segment::DeletedDocs>(); segment::DeletedDocsPtr deleted_docs = std::make_shared<segment::DeletedDocs>();
end = std::chrono::high_resolution_clock::now(); auto time2 = std::chrono::high_resolution_clock::now();
diff = end - start; std::chrono::duration<double> diff1 = time2 - time1;
ENGINE_LOG_DEBUG << "Loading uids and deleted docs took " << diff.count() << " s"; ENGINE_LOG_DEBUG << "Loading uids and deleted docs took " << diff1.count() << " s";
start = std::chrono::high_resolution_clock::now();
std::sort(ids_to_check.begin(), ids_to_check.end()); std::sort(ids_to_check.begin(), ids_to_check.end());
end = std::chrono::high_resolution_clock::now(); auto time3 = std::chrono::high_resolution_clock::now();
diff = end - start; std::chrono::duration<double> diff2 = time3 - time2;
ENGINE_LOG_DEBUG << "Sorting " << ids_to_check.size() << " ids took " << diff.count() << " s"; ENGINE_LOG_DEBUG << "Sorting " << ids_to_check.size() << " ids took " << diff2.count() << " s";
size_t delete_count = 0; size_t delete_count = 0;
auto find_diff = std::chrono::duration<double>::zero(); auto find_diff = std::chrono::duration<double>::zero();
@ -324,11 +322,13 @@ MemTable::ApplyDeletes() {
<< find_diff.count() << " s in total"; << find_diff.count() << " s in total";
ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total";
auto time4 = std::chrono::high_resolution_clock::now();
for (auto i = 0; i < indexes.size(); ++i) { for (auto i = 0; i < indexes.size(); ++i) {
indexes[i]->SetBlacklist(blacklists[i]); indexes[i]->SetBlacklist(blacklists[i]);
} }
start = std::chrono::high_resolution_clock::now(); // start = std::chrono::high_resolution_clock::now();
segment::Segment tmp_segment; segment::Segment tmp_segment;
segment::SegmentWriter segment_writer(segment_dir); segment::SegmentWriter segment_writer(segment_dir);
@ -337,21 +337,21 @@ MemTable::ApplyDeletes() {
break; break;
} }
end = std::chrono::high_resolution_clock::now(); auto time5 = std::chrono::high_resolution_clock::now();
diff = end - start; std::chrono::duration<double> diff4 = time5 - time4;
ENGINE_LOG_DEBUG << "Appended " << deleted_docs->GetSize() ENGINE_LOG_DEBUG << "Appended " << deleted_docs->GetSize()
<< " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " << diff.count() << " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " << diff4.count()
<< " s"; << " s";
start = std::chrono::high_resolution_clock::now(); // start = std::chrono::high_resolution_clock::now();
status = segment_writer.WriteBloomFilter(id_bloom_filter_ptr); status = segment_writer.WriteBloomFilter(id_bloom_filter_ptr);
if (!status.ok()) { if (!status.ok()) {
break; break;
} }
end = std::chrono::high_resolution_clock::now(); auto time6 = std::chrono::high_resolution_clock::now();
diff = end - start; std::chrono::duration<double> diff5 = time6 - time5;
ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff.count() ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff5.count()
<< " s"; << " s";
// Update table file row count // Update table file row count
@ -362,8 +362,15 @@ MemTable::ApplyDeletes() {
table_files_to_update.emplace_back(file); table_files_to_update.emplace_back(file);
} }
} }
auto time7 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff6 = time7 - time6;
diff6 = time6 - time5;
ENGINE_LOG_DEBUG << "Update table file row count in vector of segment: " << table_file.segment_id_ << " in "
<< diff6.count() << " s";
} }
auto time7 = std::chrono::high_resolution_clock::now();
status = meta_->UpdateTableFilesRowCount(table_files_to_update); status = meta_->UpdateTableFilesRowCount(table_files_to_update);
if (!status.ok()) { if (!status.ok()) {
@ -375,6 +382,8 @@ MemTable::ApplyDeletes() {
doc_ids_to_delete_.clear(); doc_ids_to_delete_.clear();
auto end_total = std::chrono::high_resolution_clock::now(); auto end_total = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff7 = end_total - time7;
ENGINE_LOG_DEBUG << "Update deletes to meta in table " << table_id_ << " in " << diff7.count() << " s";
std::chrono::duration<double> diff_total = end_total - start_total; std::chrono::duration<double> diff_total = end_total - start_total;
ENGINE_LOG_DEBUG << "Finished applying deletes in table " << table_id_ << " in " << diff_total.count() << " s"; ENGINE_LOG_DEBUG << "Finished applying deletes in table " << table_id_ << " in " << diff_total.count() << " s";