From d66d3f0d47548eaa0e1930e44c5447298e5f2e20 Mon Sep 17 00:00:00 2001 From: zhagnlu <1542303831@qq.com> Date: Sat, 22 Feb 2025 11:29:53 +0800 Subject: [PATCH] enhance:decrease dump snapshot limit from 10w to 1w (#40102) pr: #40101 Signed-off-by: luzhang Co-authored-by: luzhang --- internal/core/src/segcore/DeletedRecord.h | 2 +- internal/core/unittest/test_delete_record.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/core/src/segcore/DeletedRecord.h b/internal/core/src/segcore/DeletedRecord.h index f54392fdbb..d50331fb42 100644 --- a/internal/core/src/segcore/DeletedRecord.h +++ b/internal/core/src/segcore/DeletedRecord.h @@ -46,7 +46,7 @@ struct Comparator { using SortedDeleteList = folly::ConcurrentSkipList, Comparator>; -static int32_t DUMP_BATCH_SIZE = 100000; +static int32_t DUMP_BATCH_SIZE = 10000; static int32_t DELETE_PAIR_SIZE = sizeof(std::pair); template diff --git a/internal/core/unittest/test_delete_record.cpp b/internal/core/unittest/test_delete_record.cpp index 0755e63b91..21a23e8930 100644 --- a/internal/core/unittest/test_delete_record.cpp +++ b/internal/core/unittest/test_delete_record.cpp @@ -263,7 +263,7 @@ TEST(DeleteMVCC, snapshot) { "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2); auto i64_fid = schema->AddDebugField("age", DataType::INT64); schema->set_primary_field_id(i64_fid); - auto N = 500000; + auto N = 50000; uint64_t seg_id = 101; InsertRecord insert_record(*schema, N); DeletedRecord delete_record(&insert_record); @@ -281,7 +281,7 @@ TEST(DeleteMVCC, snapshot) { field_data->set_data_raw(insert_offset, age_data.data(), N); insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N); - auto DN = 400000; + auto DN = 40000; std::vector delete_ts(DN); std::vector delete_pk(DN); for (int i = 0; i < DN; ++i) { @@ -293,7 +293,7 @@ TEST(DeleteMVCC, snapshot) { auto snapshots = std::move(delete_record.get_snapshots()); ASSERT_EQ(3, snapshots.size()); - ASSERT_EQ(snapshots[2].second.count(), 300000); + ASSERT_EQ(snapshots[2].second.count(), 30000); } TEST(DeleteMVCC, insert_after_snapshot) { @@ -306,7 +306,7 @@ TEST(DeleteMVCC, insert_after_snapshot) { "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2); auto i64_fid = schema->AddDebugField("age", DataType::INT64); schema->set_primary_field_id(i64_fid); - auto N = 110000; + auto N = 11000; uint64_t seg_id = 101; InsertRecord insert_record(*schema, N); DeletedRecord delete_record(&insert_record); @@ -326,7 +326,7 @@ TEST(DeleteMVCC, insert_after_snapshot) { insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N); // delete (0, 1), (1, 2) .... (DN, DN + 1) - auto DN = 101000; + auto DN = 10100; std::vector delete_ts(DN); std::vector delete_pk(DN); for (int i = 0; i < DN; ++i) { @@ -338,7 +338,7 @@ TEST(DeleteMVCC, insert_after_snapshot) { auto snapshots = std::move(delete_record.get_snapshots()); ASSERT_EQ(1, snapshots.size()); - ASSERT_EQ(snapshots[0].second.count(), 100000); + ASSERT_EQ(snapshots[0].second.count(), 10000); // Query at N+1 ts {