milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce cpp unittest time cost (#8403) 

Signed-off-by: yhmo <yihua.mo@zilliz.com>
pull/8409/head
groot 2021-09-23 16:55:54 +08:00 committed by GitHub
parent 40beed9d10
commit 8b81ceb5d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 81 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
internal/core/unittest/test_c_api.cpp
View File

@ -37,6 +37,7 @@ using namespace milvus::knowhere;
namespace {
const int DIM = 16;
const int64_t ROW_COUNT = 100 * 1000;
const char*
get_default_schema_config() {
@ -706,10 +707,10 @@ TEST(CApiTest, Indexing_Without_Predicate) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
int64_t offset;
PreInsert(segment, N, &offset);
@ -829,10 +830,10 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
int64_t offset;
PreInsert(segment, N, &offset);
@ -947,10 +948,10 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
int64_t offset;
PreInsert(segment, N, &offset);
@ -964,8 +965,8 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
{
"range": {
"counter": {
"GE": 420000,
"LT": 420010
"GE": 42000,
"LT": 42010
}
}
},
@ -1060,7 +1061,7 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1083,10 +1084,10 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
{
int64_t offset;
@ -1109,7 +1110,7 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
>
op: GreaterEqual
value: <
int64_val: 420000
int64_val: 42000
>
>
>
@ -1121,7 +1122,7 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
>
op: LessThan
value: <
int64_val: 420010
int64_val: 42010
>
>
>
@ -1211,7 +1212,7 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1234,10 +1235,10 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1251,7 +1252,7 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
{
"term": {
"counter": {
"values": [420000, 420001, 420002, 420003, 420004]
"values": [42000, 42001, 42002, 42003, 42004]
}
}
},
@ -1346,7 +1347,7 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1369,10 +1370,10 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1390,19 +1391,19 @@ vector_anns: <
data_type: Int64
>
values: <
int64_val: 420000
int64_val: 42000
>
values: <
int64_val: 420001
int64_val: 42001
>
values: <
int64_val: 420002
int64_val: 42002
>
values: <
int64_val: 420003
int64_val: 42003
>
values: <
int64_val: 420004
int64_val: 42004
>
>
>
@ -1490,7 +1491,7 @@ vector_anns: <
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1513,10 +1514,10 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<uint8_t>(0);
auto query_ptr = vec_col.data() + 420000 * DIM / 8;
auto query_ptr = vec_col.data() + 42000 * DIM / 8;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1530,8 +1531,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
{
"range": {
"counter": {
"GE": 420000,
"LT": 420010
"GE": 42000,
"LT": 42010
}
}
},
@ -1627,7 +1628,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1650,10 +1651,10 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<uint8_t>(0);
auto query_ptr = vec_col.data() + 420000 * DIM / 8;
auto query_ptr = vec_col.data() + 42000 * DIM / 8;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1674,7 +1675,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
>
op: GreaterEqual
value: <
int64_val: 420000
int64_val: 42000
>
>
>
@ -1686,7 +1687,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
>
op: LessThan
value: <
int64_val: 420010
int64_val: 42010
>
>
>
@ -1777,7 +1778,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1800,10 +1801,10 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<uint8_t>(0);
auto query_ptr = vec_col.data() + 420000 * DIM / 8;
auto query_ptr = vec_col.data() + 42000 * DIM / 8;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1817,7 +1818,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
{
"term": {
"counter": {
"values": [420000, 420001, 420002, 420003, 420004]
"values": [42000, 42001, 42002, 42003, 42004]
}
}
},
@ -1918,7 +1919,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -1941,10 +1942,10 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Growing);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<uint8_t>(0);
auto query_ptr = vec_col.data() + 420000 * DIM / 8;
auto query_ptr = vec_col.data() + 42000 * DIM / 8;
int64_t offset;
PreInsert(segment, N, &offset);
@ -1961,19 +1962,19 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
data_type: Int64
>
values: <
int64_val: 420000
int64_val: 42000
>
values: <
int64_val: 420001
int64_val: 42001
>
values: <
int64_val: 420002
int64_val: 42002
>
values: <
int64_val: 420003
int64_val: 42003
>
values: <
int64_val: 420004
int64_val: 42004
>
>
>
@ -2067,7 +2068,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset],
search_result_on_raw_index->result_distances_[offset]);
}
@ -2136,11 +2137,11 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Sealed);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto counter_col = dataset.get_col<int64_t>(1);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
const char* dsl_string = R"({
"bool": {
@ -2148,8 +2149,8 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
{
"range": {
"counter": {
"GE": 420000,
"LT": 420010
"GE": 42000,
"LT": 42010
}
}
},
@ -2269,7 +2270,7 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
}
DeleteLoadIndexInfo(c_load_index_info);
@ -2288,11 +2289,11 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
auto schema = ((segcore::Collection*)collection)->get_schema();
auto segment = NewSegment(collection, 0, Sealed);
auto N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto counter_col = dataset.get_col<int64_t>(1);
auto query_ptr = vec_col.data() + 420000 * DIM;
auto query_ptr = vec_col.data() + 42000 * DIM;
const char* serialized_expr_plan = R"(vector_anns: <
field_id: 100
@ -2307,7 +2308,7 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
>
op: GreaterEqual
value: <
int64_val: 420000
int64_val: 42000
>
>
>
@ -2319,7 +2320,7 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
>
op: LessThan
value: <
int64_val: 420010
int64_val: 42010
>
>
>
@ -2435,7 +2436,7 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
auto search_result_on_bigIndex = (*(SearchResult*)c_search_result_on_bigIndex);
for (int i = 0; i < num_queries; ++i) {
auto offset = i * TOPK;
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 42000 + i);
}
DeleteLoadIndexInfo(c_load_index_info);

2
internal/core/unittest/test_concurrent_vector.cpp
View File

@ -68,7 +68,7 @@ TEST(ConcurrentVector, TestMultithreads) {
std::default_random_engine e(42 + thread_id);
int64_t data = 0;
int64_t total_count = 0;
for (int i = 0; i < 10000; ++i) {
for (int i = 0; i < 2000; ++i) {
int insert_size = e() % 150;
vector<int64_t> vec(insert_size * dim);
for (auto& x : vec) {

6
internal/core/unittest/test_expr.cpp
View File

@ -296,7 +296,7 @@ TEST(Expr, TestRange) {
schema->AddDebugField("age", DataType::INT32);
auto seg = CreateGrowingSegment(schema);
int N = 10000;
int N = 1000;
std::vector<int> age_col;
int num_iters = 100;
for (int iter = 0; iter < num_iters; ++iter) {
@ -378,7 +378,7 @@ TEST(Expr, TestTerm) {
schema->AddDebugField("age", DataType::INT32);
auto seg = CreateGrowingSegment(schema);
int N = 10000;
int N = 1000;
std::vector<int> age_col;
int num_iters = 100;
for (int iter = 0; iter < num_iters; ++iter) {
@ -550,7 +550,7 @@ TEST(Expr, TestCompare) {
schema->AddDebugField("age2", DataType::INT64);
auto seg = CreateGrowingSegment(schema);
int N = 10000;
int N = 1000;
std::vector<int> age1_col;
std::vector<int64_t> age2_col;
int num_iters = 100;

2
internal/core/unittest/test_index_wrapper.cpp
View File

@ -278,7 +278,7 @@ TEST(IVFFLATNMWrapper, Build) {
}
TEST(IVFFLATNMWrapper, Codec) {
int64_t flat_nb = 1000000;
int64_t flat_nb = 100000;
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::L2;
indexcgo::TypeParams type_params;

2
internal/core/unittest/test_indexing.cpp
View File

@ -201,7 +201,7 @@ TEST(Indexing, IVFFlatNM) {
constexpr auto DIM = 16;
constexpr auto K = 10;
auto N = 1024 * 1024 * 10;
auto N = 1024 * 1024;
auto num_query = 100;
Timer timer;
auto [raw_data, timestamps, uids] = generate_data<DIM>(N);

18
internal/core/unittest/test_sealed.cpp
View File

@ -24,6 +24,10 @@ using namespace milvus;
using namespace milvus::segcore;
using namespace milvus::query;
namespace {
const int64_t ROW_COUNT = 100 * 1000;
}
TEST(Sealed, without_predicate) {
using namespace milvus::query;
using namespace milvus::segcore;
@ -52,7 +56,7 @@ TEST(Sealed, without_predicate) {
}
})";
int64_t N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
@ -133,8 +137,8 @@ TEST(Sealed, with_predicate) {
{
"range": {
"counter": {
"GE": 420000,
"LT": 420005
"GE": 42000,
"LT": 42005
}
}
},
@ -154,11 +158,11 @@ TEST(Sealed, with_predicate) {
}
})";
int64_t N = 1000 * 1000;
auto N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto vec_col = dataset.get_col<float>(0);
auto query_ptr = vec_col.data() + 420000 * dim;
auto query_ptr = vec_col.data() + 42000 * dim;
auto segment = CreateGrowingSegment(schema);
segment->PreInsert(N);
segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
@ -205,7 +209,7 @@ TEST(Sealed, with_predicate) {
auto post_sr = sr;
for (int i = 0; i < num_queries; ++i) {
auto offset = i * topK;
ASSERT_EQ(post_sr.internal_seg_offsets_[offset], 420000 + i);
ASSERT_EQ(post_sr.internal_seg_offsets_[offset], 42000 + i);
ASSERT_EQ(post_sr.result_distances_[offset], 0.0);
}
}
@ -213,7 +217,7 @@ TEST(Sealed, with_predicate) {
TEST(Sealed, LoadFieldData) {
auto dim = 16;
auto topK = 5;
int64_t N = 1000 * 1000;
auto N = 1000 * 1000;
auto metric_type = MetricType::METRIC_L2;
auto schema = std::make_shared<Schema>();
auto fakevec_id = schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, dim, metric_type);

2
internal/core/unittest/test_segcore.cpp
View File

@ -108,6 +108,4 @@ TEST(SegmentCoreTest, SmallIndex) {
auto schema = std::make_shared<Schema>();
schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 16, MetricType::METRIC_L2);
schema->AddDebugField("age", DataType::INT32);
int N = 1024 * 1024;
auto data = DataGen(schema, N);
}

6
internal/core/unittest/test_span.cpp
View File

@ -14,11 +14,15 @@
#include "test_utils/DataGen.h"
#include "segcore/SegmentGrowing.h"
namespace {
const int64_t ROW_COUNT = 100 * 1000;
}
TEST(Span, Naive) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
int64_t N = 1000 * 1000;
int64_t N = ROW_COUNT;
constexpr int64_t size_per_chunk = 32 * 1024;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("binaryvec", DataType::VECTOR_BINARY, 512, MetricType::METRIC_Jaccard);