mirror of https://github.com/milvus-io/milvus.git
Merge branch 'branch-0.3.0' into 'branch-0.3.0'
MS-129 Add more unitest See merge request megasearch/vecwise_engine!129 Former-commit-id: 16b244522bd14fc656571c3acbe0218fb7adead5pull/191/head
commit
9b1128c24a
|
@ -4,7 +4,6 @@
|
|||
* Proprietary and confidential.
|
||||
******************************************************************************/
|
||||
#include "ClientProxy.h"
|
||||
#include "util/ConvertUtil.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
|
|
|
@ -1,44 +0,0 @@
|
|||
/*******************************************************************************
|
||||
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
||||
* Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
* Proprietary and confidential.
|
||||
******************************************************************************/
|
||||
#include "ConvertUtil.h"
|
||||
#include "Exception.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
namespace milvus {
|
||||
|
||||
static const std::string INDEX_RAW = "raw";
|
||||
static const std::string INDEX_IVFFLAT = "ivfflat";
|
||||
|
||||
std::string ConvertUtil::IndexType2Str(IndexType index) {
|
||||
static const std::map<IndexType, std::string> s_index2str = {
|
||||
{IndexType::cpu_idmap, INDEX_RAW},
|
||||
{IndexType::gpu_ivfflat, INDEX_IVFFLAT}
|
||||
};
|
||||
|
||||
const auto& iter = s_index2str.find(index);
|
||||
if(iter == s_index2str.end()) {
|
||||
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
|
||||
}
|
||||
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
IndexType ConvertUtil::Str2IndexType(const std::string& type) {
|
||||
static const std::map<std::string, IndexType> s_str2index = {
|
||||
{INDEX_RAW, IndexType::cpu_idmap},
|
||||
{INDEX_IVFFLAT, IndexType::gpu_ivfflat}
|
||||
};
|
||||
|
||||
const auto& iter = s_str2index.find(type);
|
||||
if(iter == s_str2index.end()) {
|
||||
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
|
||||
}
|
||||
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,18 +0,0 @@
|
|||
/*******************************************************************************
|
||||
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
||||
* Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
* Proprietary and confidential.
|
||||
******************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include "MilvusApi.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
class ConvertUtil {
|
||||
public:
|
||||
static std::string IndexType2Str(IndexType index);
|
||||
static IndexType Str2IndexType(const std::string& type);
|
||||
};
|
||||
|
||||
}
|
|
@ -1,159 +1,159 @@
|
|||
#include "S3ClientWrapper.h"
|
||||
|
||||
#include <aws/s3/model/CreateBucketRequest.h>
|
||||
#include <aws/s3/model/DeleteBucketRequest.h>
|
||||
#include <aws/s3/model/PutObjectRequest.h>
|
||||
#include <aws/s3/model/GetObjectRequest.h>
|
||||
#include <aws/s3/model/DeleteObjectRequest.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace milvus {
|
||||
namespace engine {
|
||||
namespace storage {
|
||||
|
||||
Status
|
||||
S3ClientWrapper::Create(const std::string &ip_address,
|
||||
const std::string &port,
|
||||
const std::string &access_key,
|
||||
const std::string &secret_key) {
|
||||
Aws::InitAPI(options_);
|
||||
Aws::Client::ClientConfiguration cfg;
|
||||
|
||||
// TODO: ip_address need to be validated.
|
||||
|
||||
cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
|
||||
cfg.scheme = Aws::Http::Scheme::HTTP;
|
||||
cfg.verifySSL =
|
||||
false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
|
||||
client_ =
|
||||
new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
|
||||
cfg,
|
||||
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
|
||||
false);
|
||||
if (client_ == nullptr) {
|
||||
std::string error = "Can't connect server.";
|
||||
return Status::Error(error);
|
||||
} else {
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Status
|
||||
S3ClientWrapper::Close() {
|
||||
if (client_ != nullptr) {
|
||||
delete client_;
|
||||
client_ = nullptr;
|
||||
}
|
||||
Aws::ShutdownAPI(options_);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status
|
||||
S3ClientWrapper::CreateBucket(std::string& bucket_name) {
|
||||
Aws::S3::Model::CreateBucketRequest request;
|
||||
request.SetBucket(bucket_name);
|
||||
|
||||
auto outcome = client_->CreateBucket(request);
|
||||
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
return Status::OK();
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "CreateBucket error: "
|
||||
<< outcome.GetError().GetExceptionName() << std::endl
|
||||
<< outcome.GetError().GetMessage() << std::endl;
|
||||
switch(outcome.GetError().GetErrorType()) {
|
||||
case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
|
||||
case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
|
||||
return Status::AlreadyExist(outcome.GetError().GetMessage());
|
||||
default:
|
||||
return Status::Error(outcome.GetError().GetMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Status
|
||||
S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
|
||||
Aws::S3::Model::DeleteBucketRequest bucket_request;
|
||||
bucket_request.SetBucket(bucket_name);
|
||||
|
||||
auto outcome = client_->DeleteBucket(bucket_request);
|
||||
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
return Status::OK();
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "DeleteBucket error: "
|
||||
<< outcome.GetError().GetExceptionName() << " - "
|
||||
<< outcome.GetError().GetMessage() << std::endl;
|
||||
return Status::Error(outcome.GetError().GetMessage());
|
||||
}
|
||||
}
|
||||
|
||||
Status
|
||||
S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
|
||||
|
||||
PutObjectRequest putObjectRequest;
|
||||
putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
|
||||
|
||||
auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
|
||||
pathkey.c_str(),
|
||||
std::ios_base::in | std::ios_base::binary);
|
||||
putObjectRequest.SetBody(input_data);
|
||||
auto put_object_result = client_->PutObject(putObjectRequest);
|
||||
if (put_object_result.IsSuccess()) {
|
||||
return Status::OK();
|
||||
} else {
|
||||
std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
|
||||
<< put_object_result.GetError().GetMessage() << std::endl;
|
||||
return Status::Error(put_object_result.GetError().GetMessage());
|
||||
}
|
||||
}
|
||||
|
||||
Status
|
||||
S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
|
||||
GetObjectRequest object_request;
|
||||
object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
|
||||
auto get_object_outcome = client_->GetObject(object_request);
|
||||
if (get_object_outcome.IsSuccess()) {
|
||||
Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
|
||||
local_file << get_object_outcome.GetResult().GetBody().rdbuf();
|
||||
return Status::OK();
|
||||
} else {
|
||||
std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
|
||||
<< get_object_outcome.GetError().GetMessage() << std::endl;
|
||||
return Status::Error(get_object_outcome.GetError().GetMessage());
|
||||
}
|
||||
}
|
||||
|
||||
Status
|
||||
S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
|
||||
Aws::S3::Model::DeleteObjectRequest object_request;
|
||||
object_request.WithBucket(bucket_name).WithKey(object_key);
|
||||
|
||||
auto delete_object_outcome = client_->DeleteObject(object_request);
|
||||
|
||||
if (delete_object_outcome.IsSuccess()) {
|
||||
return Status::OK();
|
||||
} else {
|
||||
std::cout << "DeleteObject error: " <<
|
||||
delete_object_outcome.GetError().GetExceptionName() << " " <<
|
||||
delete_object_outcome.GetError().GetMessage() << std::endl;
|
||||
|
||||
return Status::Error(delete_object_outcome.GetError().GetMessage());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//#include "S3ClientWrapper.h"
|
||||
//
|
||||
//#include <aws/s3/model/CreateBucketRequest.h>
|
||||
//#include <aws/s3/model/DeleteBucketRequest.h>
|
||||
//#include <aws/s3/model/PutObjectRequest.h>
|
||||
//#include <aws/s3/model/GetObjectRequest.h>
|
||||
//#include <aws/s3/model/DeleteObjectRequest.h>
|
||||
//
|
||||
//#include <iostream>
|
||||
//#include <fstream>
|
||||
//
|
||||
//
|
||||
//namespace zilliz {
|
||||
//namespace milvus {
|
||||
//namespace engine {
|
||||
//namespace storage {
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::Create(const std::string &ip_address,
|
||||
// const std::string &port,
|
||||
// const std::string &access_key,
|
||||
// const std::string &secret_key) {
|
||||
// Aws::InitAPI(options_);
|
||||
// Aws::Client::ClientConfiguration cfg;
|
||||
//
|
||||
// // TODO: ip_address need to be validated.
|
||||
//
|
||||
// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
|
||||
// cfg.scheme = Aws::Http::Scheme::HTTP;
|
||||
// cfg.verifySSL =
|
||||
// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
|
||||
// client_ =
|
||||
// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
|
||||
// cfg,
|
||||
// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
|
||||
// false);
|
||||
// if (client_ == nullptr) {
|
||||
// std::string error = "Can't connect server.";
|
||||
// return Status::Error(error);
|
||||
// } else {
|
||||
// return Status::OK();
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::Close() {
|
||||
// if (client_ != nullptr) {
|
||||
// delete client_;
|
||||
// client_ = nullptr;
|
||||
// }
|
||||
// Aws::ShutdownAPI(options_);
|
||||
// return Status::OK();
|
||||
//}
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
|
||||
// Aws::S3::Model::CreateBucketRequest request;
|
||||
// request.SetBucket(bucket_name);
|
||||
//
|
||||
// auto outcome = client_->CreateBucket(request);
|
||||
//
|
||||
// if (outcome.IsSuccess())
|
||||
// {
|
||||
// return Status::OK();
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// std::cout << "CreateBucket error: "
|
||||
// << outcome.GetError().GetExceptionName() << std::endl
|
||||
// << outcome.GetError().GetMessage() << std::endl;
|
||||
// switch(outcome.GetError().GetErrorType()) {
|
||||
// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
|
||||
// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
|
||||
// return Status::AlreadyExist(outcome.GetError().GetMessage());
|
||||
// default:
|
||||
// return Status::Error(outcome.GetError().GetMessage());
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
|
||||
// Aws::S3::Model::DeleteBucketRequest bucket_request;
|
||||
// bucket_request.SetBucket(bucket_name);
|
||||
//
|
||||
// auto outcome = client_->DeleteBucket(bucket_request);
|
||||
//
|
||||
// if (outcome.IsSuccess())
|
||||
// {
|
||||
// return Status::OK();
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// std::cout << "DeleteBucket error: "
|
||||
// << outcome.GetError().GetExceptionName() << " - "
|
||||
// << outcome.GetError().GetMessage() << std::endl;
|
||||
// return Status::Error(outcome.GetError().GetMessage());
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
|
||||
//
|
||||
// PutObjectRequest putObjectRequest;
|
||||
// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
|
||||
//
|
||||
// auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
|
||||
// pathkey.c_str(),
|
||||
// std::ios_base::in | std::ios_base::binary);
|
||||
// putObjectRequest.SetBody(input_data);
|
||||
// auto put_object_result = client_->PutObject(putObjectRequest);
|
||||
// if (put_object_result.IsSuccess()) {
|
||||
// return Status::OK();
|
||||
// } else {
|
||||
// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
|
||||
// << put_object_result.GetError().GetMessage() << std::endl;
|
||||
// return Status::Error(put_object_result.GetError().GetMessage());
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
|
||||
// GetObjectRequest object_request;
|
||||
// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
|
||||
// auto get_object_outcome = client_->GetObject(object_request);
|
||||
// if (get_object_outcome.IsSuccess()) {
|
||||
// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
|
||||
// local_file << get_object_outcome.GetResult().GetBody().rdbuf();
|
||||
// return Status::OK();
|
||||
// } else {
|
||||
// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
|
||||
// << get_object_outcome.GetError().GetMessage() << std::endl;
|
||||
// return Status::Error(get_object_outcome.GetError().GetMessage());
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//Status
|
||||
//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
|
||||
// Aws::S3::Model::DeleteObjectRequest object_request;
|
||||
// object_request.WithBucket(bucket_name).WithKey(object_key);
|
||||
//
|
||||
// auto delete_object_outcome = client_->DeleteObject(object_request);
|
||||
//
|
||||
// if (delete_object_outcome.IsSuccess()) {
|
||||
// return Status::OK();
|
||||
// } else {
|
||||
// std::cout << "DeleteObject error: " <<
|
||||
// delete_object_outcome.GetError().GetExceptionName() << " " <<
|
||||
// delete_object_outcome.GetError().GetMessage() << std::endl;
|
||||
//
|
||||
// return Status::Error(delete_object_outcome.GetError().GetMessage());
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//}
|
||||
//}
|
||||
//}
|
||||
//}
|
|
@ -1,45 +1,45 @@
|
|||
#pragma once
|
||||
|
||||
#include "storage/IStorage.h"
|
||||
|
||||
|
||||
#include <aws/s3/S3Client.h>
|
||||
#include <aws/core/Aws.h>
|
||||
#include <aws/core/auth/AWSCredentialsProvider.h>
|
||||
|
||||
|
||||
using namespace Aws::S3;
|
||||
using namespace Aws::S3::Model;
|
||||
|
||||
namespace zilliz {
|
||||
namespace milvus {
|
||||
namespace engine {
|
||||
namespace storage {
|
||||
|
||||
class S3ClientWrapper : public IStorage {
|
||||
public:
|
||||
|
||||
S3ClientWrapper() = default;
|
||||
~S3ClientWrapper() = default;
|
||||
|
||||
Status Create(const std::string &ip_address,
|
||||
const std::string &port,
|
||||
const std::string &access_key,
|
||||
const std::string &secret_key) override;
|
||||
Status Close() override;
|
||||
|
||||
Status CreateBucket(std::string& bucket_name) override;
|
||||
Status DeleteBucket(std::string& bucket_name) override;
|
||||
Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
|
||||
Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
|
||||
Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
|
||||
|
||||
private:
|
||||
S3Client *client_ = nullptr;
|
||||
Aws::SDKOptions options_;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//#pragma once
|
||||
//
|
||||
//#include "storage/IStorage.h"
|
||||
//
|
||||
//
|
||||
//#include <aws/s3/S3Client.h>
|
||||
//#include <aws/core/Aws.h>
|
||||
//#include <aws/core/auth/AWSCredentialsProvider.h>
|
||||
//
|
||||
//
|
||||
//using namespace Aws::S3;
|
||||
//using namespace Aws::S3::Model;
|
||||
//
|
||||
//namespace zilliz {
|
||||
//namespace milvus {
|
||||
//namespace engine {
|
||||
//namespace storage {
|
||||
//
|
||||
//class S3ClientWrapper : public IStorage {
|
||||
// public:
|
||||
//
|
||||
// S3ClientWrapper() = default;
|
||||
// ~S3ClientWrapper() = default;
|
||||
//
|
||||
// Status Create(const std::string &ip_address,
|
||||
// const std::string &port,
|
||||
// const std::string &access_key,
|
||||
// const std::string &secret_key) override;
|
||||
// Status Close() override;
|
||||
//
|
||||
// Status CreateBucket(std::string& bucket_name) override;
|
||||
// Status DeleteBucket(std::string& bucket_name) override;
|
||||
// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
|
||||
// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
|
||||
// Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
|
||||
//
|
||||
// private:
|
||||
// S3Client *client_ = nullptr;
|
||||
// Aws::SDKOptions options_;
|
||||
//};
|
||||
//
|
||||
//}
|
||||
//}
|
||||
//}
|
||||
//}
|
|
@ -11,18 +11,18 @@
|
|||
|
||||
using namespace zilliz::milvus;
|
||||
|
||||
TEST(CacheTest, CACHE_TEST) {
|
||||
cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance();
|
||||
TEST(CacheTest, CPU_CACHE_TEST) {
|
||||
cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance();
|
||||
|
||||
const int64_t gbyte = 1 << 30;
|
||||
int64_t g_num = 16;
|
||||
int64_t cap = g_num*gbyte;
|
||||
int64_t cap = g_num * gbyte;
|
||||
cpu_mgr->SetCapacity(cap);
|
||||
ASSERT_EQ(cpu_mgr->CacheCapacity(), cap);
|
||||
|
||||
const int dim = 256;
|
||||
|
||||
for(int i = 0; i < 20; i++) {
|
||||
for (int i = 0; i < 20; i++) {
|
||||
std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
|
||||
engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
|
||||
index->ntotal = 1000000;//less 1G per index
|
||||
|
@ -31,6 +31,12 @@ TEST(CacheTest, CACHE_TEST) {
|
|||
}
|
||||
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
|
||||
|
||||
auto obj = cpu_mgr->GetIndex("index_0");
|
||||
ASSERT_TRUE(obj == nullptr);
|
||||
|
||||
obj = cpu_mgr->GetIndex("index_19");
|
||||
ASSERT_TRUE(obj != nullptr);
|
||||
|
||||
{
|
||||
std::string item = "index_15";
|
||||
ASSERT_TRUE(cpu_mgr->ItemExists(item));
|
||||
|
@ -49,4 +55,27 @@ TEST(CacheTest, CACHE_TEST) {
|
|||
cpu_mgr->InsertItem("index_6g", index);
|
||||
ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully
|
||||
}
|
||||
|
||||
cpu_mgr->PrintInfo();
|
||||
}
|
||||
|
||||
TEST(CacheTest, GPU_CACHE_TEST) {
|
||||
cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance();
|
||||
|
||||
const int dim = 256;
|
||||
|
||||
for(int i = 0; i < 20; i++) {
|
||||
std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
|
||||
engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
|
||||
index->ntotal = 1000;
|
||||
|
||||
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index);
|
||||
|
||||
gpu_mgr->InsertItem("index_" + std::to_string(i), obj);
|
||||
}
|
||||
|
||||
auto obj = gpu_mgr->GetItem("index_0");
|
||||
|
||||
gpu_mgr->ClearCache();
|
||||
ASSERT_EQ(gpu_mgr->ItemCount(), 0);
|
||||
}
|
|
@ -0,0 +1,77 @@
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
||||
// Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
// Proprietary and confidential.
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "config/IConfigMgr.h"
|
||||
#include "server/ServerConfig.h"
|
||||
|
||||
using namespace zilliz::milvus;
|
||||
|
||||
static const std::string CONFIG_FILE_PATH = "../../../conf/server_config.yaml";
|
||||
|
||||
TEST(ConfigTest, CONFIG_TEST) {
|
||||
server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance();
|
||||
server::ServerError err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
|
||||
ASSERT_EQ(err, server::SERVER_SUCCESS);
|
||||
|
||||
server::ConfigNode& root_config = config_mgr->GetRootNode();
|
||||
server::ConfigNode& server_config = root_config.GetChild("server_config");
|
||||
server::ConfigNode& db_config = root_config.GetChild("db_config");
|
||||
server::ConfigNode& metric_config = root_config.GetChild("metric_config");
|
||||
server::ConfigNode& cache_config = root_config.GetChild("cache_config");
|
||||
|
||||
std::string address = server_config.GetValue("address");
|
||||
ASSERT_TRUE(!address.empty());
|
||||
int64_t port = server_config.GetInt64Value("port");
|
||||
ASSERT_TRUE(port != 0);
|
||||
|
||||
server_config.SetValue("test", "2.5");
|
||||
double test = server_config.GetDoubleValue("test");
|
||||
ASSERT_EQ(test, 2.5);
|
||||
|
||||
server::ConfigNode fake;
|
||||
server_config.AddChild("fake", fake);
|
||||
fake = server_config.GetChild("fake");
|
||||
server::ConfigNodeArr arr;
|
||||
server_config.GetChildren(arr);
|
||||
ASSERT_EQ(arr.size(), 1UL);
|
||||
|
||||
server_config.ClearChildren();
|
||||
auto children = server_config.GetChildren();
|
||||
ASSERT_TRUE(children.empty());
|
||||
|
||||
root_config.PrintAll();
|
||||
std::string all = root_config.DumpString();
|
||||
ASSERT_TRUE(!all.empty());
|
||||
|
||||
server_config.ClearConfig();
|
||||
auto configs = server_config.GetConfig();
|
||||
ASSERT_TRUE(configs.empty());
|
||||
|
||||
server_config.AddSequenceItem("seq", "aaa");
|
||||
server_config.AddSequenceItem("seq", "bbb");
|
||||
auto seq = server_config.GetSequence("seq");
|
||||
ASSERT_EQ(seq.size(), 2UL);
|
||||
|
||||
server_config.ClearSequences();
|
||||
auto seqs = server_config.GetSequences();
|
||||
ASSERT_TRUE(seqs.empty());
|
||||
}
|
||||
|
||||
TEST(ConfigTest, SERVER_CONFIG_TEST) {
|
||||
server::ServerConfig& config = server::ServerConfig::GetInstance();
|
||||
server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH);
|
||||
ASSERT_EQ(err, server::SERVER_SUCCESS);
|
||||
|
||||
server::ConfigNode node1 = config.GetConfig("server_config");
|
||||
server::ConfigNode& node2 = config.GetConfig("cache_config");
|
||||
node1.Combine(node2);
|
||||
|
||||
int32_t cap = node1.GetInt32Value("cpu_cache_capacity");
|
||||
ASSERT_GT(cap, 0);
|
||||
|
||||
config.PrintAll();
|
||||
}
|
|
@ -1,66 +1,66 @@
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
||||
// Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
// Proprietary and confidential.
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "storage/IStorage.h"
|
||||
#include "storage/s3/S3ClientWrapper.h"
|
||||
#include <gtest/gtest.h>
|
||||
#include <memory.h>
|
||||
#include <fstream>
|
||||
|
||||
|
||||
using namespace zilliz::milvus::engine;
|
||||
|
||||
TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
|
||||
|
||||
std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
|
||||
|
||||
std::string ip_address = "127.0.0.1";
|
||||
std::string port = "9000";
|
||||
std::string access_key = "AKIAIOSFODNN7EXAMPLE";
|
||||
std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
|
||||
|
||||
Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
std::string filename = "/tmp/s3_test_file";
|
||||
std::string bucket_name = "bucktname";
|
||||
std::string object_name = "test_file";
|
||||
|
||||
status = storage_ptr->CreateBucket(bucket_name);
|
||||
std::cout << status.IsAlreadyExist() << std::endl;
|
||||
if (status.IsAlreadyExist()) {
|
||||
status = storage_ptr->DeleteBucket(bucket_name);
|
||||
status = storage_ptr->CreateBucket(bucket_name);
|
||||
}
|
||||
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
std::ofstream ofile(filename);
|
||||
std::stringstream ss;
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
ss << i;
|
||||
}
|
||||
ofile << ss.str() << std::endl;
|
||||
ofile.close();
|
||||
status = storage_ptr->UploadFile(bucket_name, object_name, filename);
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
|
||||
std::ifstream infile(filename);
|
||||
std::string in_buffer;
|
||||
infile >> in_buffer;
|
||||
ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
|
||||
|
||||
status = storage_ptr->DeleteFile(bucket_name, object_name);
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
status = storage_ptr->DeleteBucket(bucket_name);
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
status = storage_ptr->Close();
|
||||
ASSERT_TRUE(status.ok());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
|
||||
//// Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
//// Proprietary and confidential.
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
//#include "storage/IStorage.h"
|
||||
//#include "storage/s3/S3ClientWrapper.h"
|
||||
//#include <gtest/gtest.h>
|
||||
//#include <memory.h>
|
||||
//#include <fstream>
|
||||
//
|
||||
//
|
||||
//using namespace zilliz::milvus::engine;
|
||||
//
|
||||
//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
|
||||
//
|
||||
// std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
|
||||
//
|
||||
// std::string ip_address = "127.0.0.1";
|
||||
// std::string port = "9000";
|
||||
// std::string access_key = "AKIAIOSFODNN7EXAMPLE";
|
||||
// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
|
||||
//
|
||||
// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//
|
||||
// std::string filename = "/tmp/s3_test_file";
|
||||
// std::string bucket_name = "bucktname";
|
||||
// std::string object_name = "test_file";
|
||||
//
|
||||
// status = storage_ptr->CreateBucket(bucket_name);
|
||||
// std::cout << status.IsAlreadyExist() << std::endl;
|
||||
// if (status.IsAlreadyExist()) {
|
||||
// status = storage_ptr->DeleteBucket(bucket_name);
|
||||
// status = storage_ptr->CreateBucket(bucket_name);
|
||||
// }
|
||||
//
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//
|
||||
// std::ofstream ofile(filename);
|
||||
// std::stringstream ss;
|
||||
// for (int i = 0; i < 1024; ++i) {
|
||||
// ss << i;
|
||||
// }
|
||||
// ofile << ss.str() << std::endl;
|
||||
// ofile.close();
|
||||
// status = storage_ptr->UploadFile(bucket_name, object_name, filename);
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//
|
||||
// status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
|
||||
// std::ifstream infile(filename);
|
||||
// std::string in_buffer;
|
||||
// infile >> in_buffer;
|
||||
// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
|
||||
//
|
||||
// status = storage_ptr->DeleteFile(bucket_name, object_name);
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//
|
||||
// status = storage_ptr->DeleteBucket(bucket_name);
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//
|
||||
// status = storage_ptr->Close();
|
||||
// ASSERT_TRUE(status.ok());
|
||||
//}
|
||||
//
|
||||
|
|
Loading…
Reference in New Issue