mirror of https://github.com/milvus-io/milvus.git
246 lines
7.5 KiB
C++
246 lines
7.5 KiB
C++
////////////////////////////////////////////////////////////////////////////////
|
|
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
|
// Unauthorized copying of this file, via any medium is strictly prohibited.
|
|
// Proprietary and confidential.
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
#include <gtest/gtest.h>
|
|
#include <thread>
|
|
#include <easylogging++.h>
|
|
#include <stdlib.h>
|
|
#include <time.h>
|
|
|
|
#include "utils.h"
|
|
#include "db/DBMetaImpl.h"
|
|
#include "db/Factories.h"
|
|
#include "db/Utils.h"
|
|
#include "db/MetaConsts.h"
|
|
|
|
using namespace zilliz::vecwise::engine;
|
|
|
|
TEST_F(MetaTest, GROUP_TEST) {
|
|
auto group_id = "meta_test_group";
|
|
|
|
meta::TableSchema group;
|
|
group.group_id = group_id;
|
|
auto status = impl_->add_group(group);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
auto gid = group.id;
|
|
group.id = -1;
|
|
status = impl_->get_group(group);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(group.id, gid);
|
|
ASSERT_EQ(group.group_id, group_id);
|
|
|
|
group.group_id = "not_found";
|
|
status = impl_->get_group(group);
|
|
ASSERT_TRUE(!status.ok());
|
|
|
|
group.group_id = group_id;
|
|
status = impl_->add_group(group);
|
|
ASSERT_TRUE(!status.ok());
|
|
}
|
|
|
|
TEST_F(MetaTest, GROUP_FILE_TEST) {
|
|
auto group_id = "meta_test_group";
|
|
|
|
meta::TableSchema group;
|
|
group.group_id = group_id;
|
|
auto status = impl_->add_group(group);
|
|
|
|
meta::TableFileSchema group_file;
|
|
group_file.group_id = group.group_id;
|
|
status = impl_->add_group_file(group_file);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW);
|
|
|
|
auto file_id = group_file.file_id;
|
|
|
|
auto new_file_type = meta::TableFileSchema::INDEX;
|
|
group_file.file_type = new_file_type;
|
|
|
|
status = impl_->update_group_file(group_file);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(group_file.file_type, new_file_type);
|
|
|
|
meta::DatesT dates;
|
|
dates.push_back(meta::Meta::GetDate());
|
|
status = impl_->delete_group_partitions(group_file.group_id, dates);
|
|
ASSERT_FALSE(status.ok());
|
|
|
|
dates.clear();
|
|
for (auto i=2; i < 10; ++i) {
|
|
dates.push_back(meta::Meta::GetDateWithDelta(-1*i));
|
|
}
|
|
status = impl_->delete_group_partitions(group_file.group_id, dates);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
group_file.date = meta::Meta::GetDateWithDelta(-2);
|
|
status = impl_->update_group_file(group_file);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2));
|
|
ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
|
|
|
|
dates.clear();
|
|
dates.push_back(group_file.date);
|
|
status = impl_->delete_group_partitions(group_file.group_id, dates);
|
|
ASSERT_TRUE(status.ok());
|
|
status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
|
|
}
|
|
|
|
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
|
|
srand(time(0));
|
|
DBMetaOptions options;
|
|
options.path = "/tmp/vecwise_test";
|
|
int days_num = rand() % 100;
|
|
std::stringstream ss;
|
|
ss << "days:" << days_num;
|
|
options.archive_conf = ArchiveConf("delete", ss.str());
|
|
|
|
auto impl = meta::DBMetaImpl(options);
|
|
auto group_id = "meta_test_group";
|
|
|
|
meta::TableSchema group;
|
|
group.group_id = group_id;
|
|
auto status = impl.add_group(group);
|
|
|
|
meta::GroupFilesSchema files;
|
|
meta::TableFileSchema group_file;
|
|
group_file.group_id = group.group_id;
|
|
|
|
auto cnt = 100;
|
|
long ts = utils::GetMicroSecTimeStamp();
|
|
std::vector<int> days;
|
|
for (auto i=0; i<cnt; ++i) {
|
|
status = impl.add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::NEW;
|
|
int day = rand() % (days_num*2);
|
|
group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
|
|
status = impl.update_group_file(group_file);
|
|
files.push_back(group_file);
|
|
days.push_back(day);
|
|
}
|
|
|
|
impl.archive_files();
|
|
int i = 0;
|
|
|
|
for (auto file : files) {
|
|
status = impl.get_group_file(file.group_id, file.file_id, file);
|
|
ASSERT_TRUE(status.ok());
|
|
if (days[i] < days_num) {
|
|
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
|
|
} else {
|
|
ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
|
|
}
|
|
i++;
|
|
}
|
|
|
|
impl.drop_all();
|
|
}
|
|
|
|
TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
|
|
DBMetaOptions options;
|
|
options.path = "/tmp/vecwise_test";
|
|
options.archive_conf = ArchiveConf("delete", "disk:11");
|
|
|
|
auto impl = meta::DBMetaImpl(options);
|
|
auto group_id = "meta_test_group";
|
|
|
|
meta::TableSchema group;
|
|
group.group_id = group_id;
|
|
auto status = impl.add_group(group);
|
|
|
|
meta::GroupFilesSchema files;
|
|
meta::TableFileSchema group_file;
|
|
group_file.group_id = group.group_id;
|
|
|
|
auto cnt = 10;
|
|
auto each_size = 2UL;
|
|
for (auto i=0; i<cnt; ++i) {
|
|
status = impl.add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::NEW;
|
|
group_file.size = each_size * meta::G;
|
|
status = impl.update_group_file(group_file);
|
|
files.push_back(group_file);
|
|
}
|
|
|
|
impl.archive_files();
|
|
int i = 0;
|
|
|
|
for (auto file : files) {
|
|
status = impl.get_group_file(file.group_id, file.file_id, file);
|
|
ASSERT_TRUE(status.ok());
|
|
if (i < 5) {
|
|
ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
|
|
} else {
|
|
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
|
|
}
|
|
++i;
|
|
}
|
|
|
|
impl.drop_all();
|
|
}
|
|
|
|
TEST_F(MetaTest, GROUP_FILES_TEST) {
|
|
auto group_id = "meta_test_group";
|
|
|
|
meta::TableSchema group;
|
|
group.group_id = group_id;
|
|
auto status = impl_->add_group(group);
|
|
|
|
int new_files_cnt = 4;
|
|
int raw_files_cnt = 5;
|
|
int to_index_files_cnt = 6;
|
|
int index_files_cnt = 7;
|
|
|
|
meta::TableFileSchema group_file;
|
|
group_file.group_id = group.group_id;
|
|
|
|
for (auto i=0; i<new_files_cnt; ++i) {
|
|
status = impl_->add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::NEW;
|
|
status = impl_->update_group_file(group_file);
|
|
}
|
|
|
|
for (auto i=0; i<raw_files_cnt; ++i) {
|
|
status = impl_->add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::RAW;
|
|
status = impl_->update_group_file(group_file);
|
|
}
|
|
|
|
for (auto i=0; i<to_index_files_cnt; ++i) {
|
|
status = impl_->add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::TO_INDEX;
|
|
status = impl_->update_group_file(group_file);
|
|
}
|
|
|
|
for (auto i=0; i<index_files_cnt; ++i) {
|
|
status = impl_->add_group_file(group_file);
|
|
group_file.file_type = meta::TableFileSchema::INDEX;
|
|
status = impl_->update_group_file(group_file);
|
|
}
|
|
|
|
meta::GroupFilesSchema files;
|
|
|
|
status = impl_->files_to_index(files);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(files.size(), to_index_files_cnt);
|
|
|
|
meta::DatePartionedGroupFilesSchema dated_files;
|
|
status = impl_->files_to_merge(group.group_id, dated_files);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt);
|
|
|
|
status = impl_->files_to_index(files);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(files.size(), to_index_files_cnt);
|
|
|
|
meta::DatesT dates = {group_file.date};
|
|
status = impl_->files_to_search(group_id, dates, dated_files);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(dated_files[group_file.date].size(),
|
|
to_index_files_cnt+raw_files_cnt+index_files_cnt);
|
|
}
|