milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance:decrease dump snapshot limit from 10w to 1w (#40102) 

pr: #40101

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
pull/40108/head
zhagnlu 2025-02-22 11:29:53 +08:00 committed by GitHub
parent be30f5301d
commit d66d3f0d47
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/segcore/DeletedRecord.h
View File

@ -46,7 +46,7 @@ struct Comparator {
using SortedDeleteList =
folly::ConcurrentSkipList<std::pair<Timestamp, Offset>, Comparator>;
static int32_t DUMP_BATCH_SIZE = 100000;
static int32_t DUMP_BATCH_SIZE = 10000;
static int32_t DELETE_PAIR_SIZE = sizeof(std::pair<Timestamp, Offset>);
template <bool is_sealed = false>

12
internal/core/unittest/test_delete_record.cpp
View File

@ -263,7 +263,7 @@ TEST(DeleteMVCC, snapshot) {
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
auto i64_fid = schema->AddDebugField("age", DataType::INT64);
schema->set_primary_field_id(i64_fid);
auto N = 500000;
auto N = 50000;
uint64_t seg_id = 101;
InsertRecord insert_record(*schema, N);
DeletedRecord<false> delete_record(&insert_record);
@ -281,7 +281,7 @@ TEST(DeleteMVCC, snapshot) {
field_data->set_data_raw(insert_offset, age_data.data(), N);
insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N);
auto DN = 400000;
auto DN = 40000;
std::vector<Timestamp> delete_ts(DN);
std::vector<PkType> delete_pk(DN);
for (int i = 0; i < DN; ++i) {
@ -293,7 +293,7 @@ TEST(DeleteMVCC, snapshot) {
auto snapshots = std::move(delete_record.get_snapshots());
ASSERT_EQ(3, snapshots.size());
ASSERT_EQ(snapshots[2].second.count(), 300000);
ASSERT_EQ(snapshots[2].second.count(), 30000);
}
TEST(DeleteMVCC, insert_after_snapshot) {
@ -306,7 +306,7 @@ TEST(DeleteMVCC, insert_after_snapshot) {
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
auto i64_fid = schema->AddDebugField("age", DataType::INT64);
schema->set_primary_field_id(i64_fid);
auto N = 110000;
auto N = 11000;
uint64_t seg_id = 101;
InsertRecord insert_record(*schema, N);
DeletedRecord<false> delete_record(&insert_record);
@ -326,7 +326,7 @@ TEST(DeleteMVCC, insert_after_snapshot) {
insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N);
// delete (0, 1), (1, 2) .... (DN, DN + 1)
auto DN = 101000;
auto DN = 10100;
std::vector<Timestamp> delete_ts(DN);
std::vector<PkType> delete_pk(DN);
for (int i = 0; i < DN; ++i) {
@ -338,7 +338,7 @@ TEST(DeleteMVCC, insert_after_snapshot) {
auto snapshots = std::move(delete_record.get_snapshots());
ASSERT_EQ(1, snapshots.size());
ASSERT_EQ(snapshots[0].second.count(), 100000);
ASSERT_EQ(snapshots[0].second.count(), 10000);
// Query at N+1 ts
{