mirror of https://github.com/milvus-io/milvus.git
Remove unused codes (#732)
* remove unused function and comments * remove always-true parameter * run on python2 explicitly * fix clang-formatpull/729/head
parent
ce1b42c651
commit
4847e5027f
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python2
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
# or more contributor license agreements. See the NOTICE file
|
# or more contributor license agreements. See the NOTICE file
|
||||||
# distributed with this work for additional information
|
# distributed with this work for additional information
|
||||||
|
|
|
@ -21,14 +21,13 @@ namespace milvus {
|
||||||
namespace scheduler {
|
namespace scheduler {
|
||||||
|
|
||||||
std::shared_ptr<Resource>
|
std::shared_ptr<Resource>
|
||||||
ResourceFactory::Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader,
|
ResourceFactory::Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_executor) {
|
||||||
bool enable_executor) {
|
|
||||||
if (type == "DISK") {
|
if (type == "DISK") {
|
||||||
return std::make_shared<DiskResource>(name, device_id, enable_loader, enable_executor);
|
return std::make_shared<DiskResource>(name, device_id, enable_executor);
|
||||||
} else if (type == "CPU") {
|
} else if (type == "CPU") {
|
||||||
return std::make_shared<CpuResource>(name, device_id, enable_loader, enable_executor);
|
return std::make_shared<CpuResource>(name, device_id, enable_executor);
|
||||||
} else if (type == "GPU") {
|
} else if (type == "GPU") {
|
||||||
return std::make_shared<GpuResource>(name, device_id, enable_loader, enable_executor);
|
return std::make_shared<GpuResource>(name, device_id, enable_executor);
|
||||||
} else {
|
} else {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,8 +31,7 @@ namespace scheduler {
|
||||||
class ResourceFactory {
|
class ResourceFactory {
|
||||||
public:
|
public:
|
||||||
static std::shared_ptr<Resource>
|
static std::shared_ptr<Resource>
|
||||||
Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader = true,
|
Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_executor = true);
|
||||||
bool enable_executor = true);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace scheduler
|
} // namespace scheduler
|
||||||
|
|
|
@ -46,10 +46,10 @@ std::mutex BuildMgrInst::mutex_;
|
||||||
void
|
void
|
||||||
load_simple_config() {
|
load_simple_config() {
|
||||||
// create and connect
|
// create and connect
|
||||||
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
|
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, false));
|
||||||
|
|
||||||
auto io = Connection("io", 500);
|
auto io = Connection("io", 500);
|
||||||
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));
|
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0));
|
||||||
ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
|
ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
|
||||||
|
|
||||||
// get resources
|
// get resources
|
||||||
|
@ -79,13 +79,12 @@ load_simple_config() {
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& gpu_id : gpu_ids) {
|
for (auto& gpu_id : gpu_ids) {
|
||||||
ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
|
ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id));
|
||||||
ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
|
ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& not_find_id : not_find_build_ids) {
|
for (auto& not_find_id : not_find_build_ids) {
|
||||||
ResMgrInst::GetInstance()->Add(
|
ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id));
|
||||||
ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true));
|
|
||||||
ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie);
|
ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -77,6 +77,12 @@ Scheduler::Dump() const {
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
Scheduler::process(const EventPtr& event) {
|
||||||
|
auto process_event = event_register_.at(static_cast<int>(event->Type()));
|
||||||
|
process_event(event);
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Scheduler::worker_function() {
|
Scheduler::worker_function() {
|
||||||
while (running_) {
|
while (running_) {
|
||||||
|
@ -88,16 +94,10 @@ Scheduler::worker_function() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
Process(event);
|
process(event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
|
||||||
Scheduler::Process(const EventPtr& event) {
|
|
||||||
auto process_event = event_register_.at(static_cast<int>(event->Type()));
|
|
||||||
process_event(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO(wxyu): refactor the function
|
// TODO(wxyu): refactor the function
|
||||||
void
|
void
|
||||||
Scheduler::OnLoadCompleted(const EventPtr& event) {
|
Scheduler::OnLoadCompleted(const EventPtr& event) {
|
||||||
|
|
|
@ -41,21 +41,12 @@ class Scheduler : public interface::dumpable {
|
||||||
Scheduler(const Scheduler&) = delete;
|
Scheduler(const Scheduler&) = delete;
|
||||||
Scheduler(Scheduler&&) = delete;
|
Scheduler(Scheduler&&) = delete;
|
||||||
|
|
||||||
/*
|
|
||||||
* Start worker thread;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
Start();
|
Start();
|
||||||
|
|
||||||
/*
|
|
||||||
* Stop worker thread, join it;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
Stop();
|
Stop();
|
||||||
|
|
||||||
/*
|
|
||||||
* Post event to scheduler event queue;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
PostEvent(const EventPtr& event);
|
PostEvent(const EventPtr& event);
|
||||||
|
|
||||||
|
@ -63,55 +54,22 @@ class Scheduler : public interface::dumpable {
|
||||||
Dump() const override;
|
Dump() const override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/******** Events ********/
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Process start up events;
|
|
||||||
*
|
|
||||||
* Actions:
|
|
||||||
* Pull task from neighbours;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
OnStartUp(const EventPtr& event);
|
OnStartUp(const EventPtr& event);
|
||||||
|
|
||||||
/*
|
|
||||||
* Process finish task events;
|
|
||||||
*
|
|
||||||
* Actions:
|
|
||||||
* Pull task from neighbours;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
OnFinishTask(const EventPtr& event);
|
OnFinishTask(const EventPtr& event);
|
||||||
|
|
||||||
/*
|
|
||||||
* Process copy completed events;
|
|
||||||
*
|
|
||||||
* Actions:
|
|
||||||
* Mark task source MOVED;
|
|
||||||
* Pull task from neighbours;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
OnLoadCompleted(const EventPtr& event);
|
OnLoadCompleted(const EventPtr& event);
|
||||||
|
|
||||||
/*
|
|
||||||
* Process task table updated events, which happened on task_table->put;
|
|
||||||
*
|
|
||||||
* Actions:
|
|
||||||
* Push task to neighbours;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
OnTaskTableUpdated(const EventPtr& event);
|
OnTaskTableUpdated(const EventPtr& event);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/*
|
|
||||||
* Dispatch event to event handler;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
Process(const EventPtr& event);
|
process(const EventPtr& event);
|
||||||
|
|
||||||
/*
|
|
||||||
* Called by worker_thread_;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
worker_function();
|
worker_function();
|
||||||
|
|
||||||
|
|
|
@ -119,9 +119,6 @@ class TaskTable : public interface::dumpable {
|
||||||
subscriber_ = std::move(subscriber);
|
subscriber_ = std::move(subscriber);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Put one task;
|
|
||||||
*/
|
|
||||||
void
|
void
|
||||||
Put(TaskPtr task, TaskTableItemPtr from = nullptr);
|
Put(TaskPtr task, TaskTableItemPtr from = nullptr);
|
||||||
|
|
||||||
|
|
|
@ -35,14 +35,5 @@ get_current_timestamp() {
|
||||||
return millis;
|
return millis;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t
|
|
||||||
get_num_gpu() {
|
|
||||||
int n_devices = 0;
|
|
||||||
#ifdef MILVUS_GPU_VERSION
|
|
||||||
cudaGetDeviceCount(&n_devices);
|
|
||||||
#endif
|
|
||||||
return n_devices;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace scheduler
|
} // namespace scheduler
|
||||||
} // namespace milvus
|
} // namespace milvus
|
||||||
|
|
|
@ -24,8 +24,5 @@ namespace scheduler {
|
||||||
uint64_t
|
uint64_t
|
||||||
get_current_timestamp();
|
get_current_timestamp();
|
||||||
|
|
||||||
uint64_t
|
|
||||||
get_num_gpu();
|
|
||||||
|
|
||||||
} // namespace scheduler
|
} // namespace scheduler
|
||||||
} // namespace milvus
|
} // namespace milvus
|
||||||
|
|
|
@ -28,8 +28,8 @@ operator<<(std::ostream& out, const CpuResource& resource) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
|
CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_executor)
|
||||||
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {
|
: Resource(std::move(name), ResourceType::CPU, device_id, enable_executor) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
|
|
@ -26,7 +26,7 @@ namespace scheduler {
|
||||||
|
|
||||||
class CpuResource : public Resource {
|
class CpuResource : public Resource {
|
||||||
public:
|
public:
|
||||||
explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
|
explicit CpuResource(std::string name, uint64_t device_id, bool enable_executor);
|
||||||
|
|
||||||
friend std::ostream&
|
friend std::ostream&
|
||||||
operator<<(std::ostream& out, const CpuResource& resource);
|
operator<<(std::ostream& out, const CpuResource& resource);
|
||||||
|
|
|
@ -29,8 +29,8 @@ operator<<(std::ostream& out, const DiskResource& resource) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
|
DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_executor)
|
||||||
: Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) {
|
: Resource(std::move(name), ResourceType::DISK, device_id, enable_executor) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
|
|
@ -26,7 +26,7 @@ namespace scheduler {
|
||||||
|
|
||||||
class DiskResource : public Resource {
|
class DiskResource : public Resource {
|
||||||
public:
|
public:
|
||||||
explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
|
explicit DiskResource(std::string name, uint64_t device_id, bool enable_executor);
|
||||||
|
|
||||||
friend std::ostream&
|
friend std::ostream&
|
||||||
operator<<(std::ostream& out, const DiskResource& resource);
|
operator<<(std::ostream& out, const DiskResource& resource);
|
||||||
|
|
|
@ -26,8 +26,8 @@ operator<<(std::ostream& out, const GpuResource& resource) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
|
GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_executor)
|
||||||
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {
|
: Resource(std::move(name), ResourceType::GPU, device_id, enable_executor) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
|
|
@ -27,7 +27,7 @@ namespace scheduler {
|
||||||
|
|
||||||
class GpuResource : public Resource {
|
class GpuResource : public Resource {
|
||||||
public:
|
public:
|
||||||
explicit GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
|
explicit GpuResource(std::string name, uint64_t device_id, bool enable_executor);
|
||||||
|
|
||||||
friend std::ostream&
|
friend std::ostream&
|
||||||
operator<<(std::ostream& out, const GpuResource& resource);
|
operator<<(std::ostream& out, const GpuResource& resource);
|
||||||
|
|
|
@ -48,12 +48,8 @@ ToString(ResourceType type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Resource::Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor)
|
Resource::Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_executor)
|
||||||
: name_(std::move(name)),
|
: name_(std::move(name)), type_(type), device_id_(device_id), enable_executor_(enable_executor) {
|
||||||
type_(type),
|
|
||||||
device_id_(device_id),
|
|
||||||
enable_loader_(enable_loader),
|
|
||||||
enable_executor_(enable_executor) {
|
|
||||||
// register subscriber in tasktable
|
// register subscriber in tasktable
|
||||||
task_table_.RegisterSubscriber([&] {
|
task_table_.RegisterSubscriber([&] {
|
||||||
if (subscriber_) {
|
if (subscriber_) {
|
||||||
|
@ -66,9 +62,7 @@ Resource::Resource(std::string name, ResourceType type, uint64_t device_id, bool
|
||||||
void
|
void
|
||||||
Resource::Start() {
|
Resource::Start() {
|
||||||
running_ = true;
|
running_ = true;
|
||||||
if (enable_loader_) {
|
loader_thread_ = std::thread(&Resource::loader_function, this);
|
||||||
loader_thread_ = std::thread(&Resource::loader_function, this);
|
|
||||||
}
|
|
||||||
if (enable_executor_) {
|
if (enable_executor_) {
|
||||||
executor_thread_ = std::thread(&Resource::executor_function, this);
|
executor_thread_ = std::thread(&Resource::executor_function, this);
|
||||||
}
|
}
|
||||||
|
@ -77,10 +71,8 @@ Resource::Start() {
|
||||||
void
|
void
|
||||||
Resource::Stop() {
|
Resource::Stop() {
|
||||||
running_ = false;
|
running_ = false;
|
||||||
if (enable_loader_) {
|
WakeupLoader();
|
||||||
WakeupLoader();
|
loader_thread_.join();
|
||||||
loader_thread_.join();
|
|
||||||
}
|
|
||||||
if (enable_executor_) {
|
if (enable_executor_) {
|
||||||
WakeupExecutor();
|
WakeupExecutor();
|
||||||
executor_thread_.join();
|
executor_thread_.join();
|
||||||
|
@ -115,7 +107,6 @@ Resource::Dump() const {
|
||||||
{"task_total_cost", total_cost_},
|
{"task_total_cost", total_cost_},
|
||||||
{"total_tasks", total_task_},
|
{"total_tasks", total_task_},
|
||||||
{"running", running_},
|
{"running", running_},
|
||||||
{"enable_loader", enable_loader_},
|
|
||||||
{"enable_executor", enable_executor_},
|
{"enable_executor", enable_executor_},
|
||||||
};
|
};
|
||||||
return ret;
|
return ret;
|
||||||
|
|
|
@ -102,11 +102,6 @@ class Resource : public Node, public std::enable_shared_from_this<Resource> {
|
||||||
}
|
}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
inline bool
|
|
||||||
HasLoader() const {
|
|
||||||
return enable_loader_;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline bool
|
inline bool
|
||||||
HasExecutor() const {
|
HasExecutor() const {
|
||||||
return enable_executor_;
|
return enable_executor_;
|
||||||
|
@ -134,7 +129,7 @@ class Resource : public Node, public std::enable_shared_from_this<Resource> {
|
||||||
operator<<(std::ostream& out, const Resource& resource);
|
operator<<(std::ostream& out, const Resource& resource);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor);
|
Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_executor);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Implementation by inherit class;
|
* Implementation by inherit class;
|
||||||
|
@ -193,7 +188,6 @@ class Resource : public Node, public std::enable_shared_from_this<Resource> {
|
||||||
std::function<void(EventPtr)> subscriber_ = nullptr;
|
std::function<void(EventPtr)> subscriber_ = nullptr;
|
||||||
|
|
||||||
bool running_ = false;
|
bool running_ = false;
|
||||||
bool enable_loader_ = true;
|
|
||||||
bool enable_executor_ = true;
|
bool enable_executor_ = true;
|
||||||
std::thread loader_thread_;
|
std::thread loader_thread_;
|
||||||
std::thread executor_thread_;
|
std::thread executor_thread_;
|
||||||
|
|
|
@ -28,8 +28,8 @@ operator<<(std::ostream& out, const TestResource& resource) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
TestResource::TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
|
TestResource::TestResource(std::string name, uint64_t device_id, bool enable_executor)
|
||||||
: Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) {
|
: Resource(std::move(name), ResourceType::TEST, device_id, enable_executor) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
|
|
@ -27,7 +27,7 @@ namespace scheduler {
|
||||||
|
|
||||||
class TestResource : public Resource {
|
class TestResource : public Resource {
|
||||||
public:
|
public:
|
||||||
explicit TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
|
explicit TestResource(std::string name, uint64_t device_id, bool enable_executor);
|
||||||
|
|
||||||
friend std::ostream&
|
friend std::ostream&
|
||||||
operator<<(std::ostream& out, const TestResource& resource);
|
operator<<(std::ostream& out, const TestResource& resource);
|
||||||
|
|
|
@ -152,14 +152,14 @@ DBTest::SetUp() {
|
||||||
|
|
||||||
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
|
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
|
||||||
res_mgr->Clear();
|
res_mgr->Clear();
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, false));
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0));
|
||||||
|
|
||||||
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
|
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
|
||||||
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
|
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
|
||||||
res_mgr->Connect("disk", "cpu", default_conn);
|
res_mgr->Connect("disk", "cpu", default_conn);
|
||||||
#ifdef MILVUS_GPU_VERSION
|
#ifdef MILVUS_GPU_VERSION
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0, true, true));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0));
|
||||||
res_mgr->Connect("cpu", "0", PCIE);
|
res_mgr->Connect("cpu", "0", PCIE);
|
||||||
#endif
|
#endif
|
||||||
res_mgr->Start();
|
res_mgr->Start();
|
||||||
|
|
|
@ -30,8 +30,8 @@ class AlgorithmTest : public testing::Test {
|
||||||
protected:
|
protected:
|
||||||
void
|
void
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false);
|
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, false);
|
||||||
ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0, true, true);
|
ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0);
|
||||||
ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1);
|
ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1);
|
||||||
ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2);
|
ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2);
|
||||||
ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0);
|
ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0);
|
||||||
|
|
|
@ -36,69 +36,44 @@ class ResourceBaseTest : public testing::Test {
|
||||||
protected:
|
protected:
|
||||||
void
|
void
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
only_loader_ = std::make_shared<DiskResource>(name1, id1, true, false);
|
enable_executor_ = std::make_shared<CpuResource>(name1, id1, true);
|
||||||
only_executor_ = std::make_shared<CpuResource>(name2, id2, false, true);
|
disable_executor_ = std::make_shared<GpuResource>(name2, id2, false);
|
||||||
both_enable_ = std::make_shared<GpuResource>(name3, id3, true, true);
|
|
||||||
both_disable_ = std::make_shared<TestResource>(name4, id4, false, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::string name1 = "only_loader_";
|
const std::string name1 = "enable_executor_";
|
||||||
const std::string name2 = "only_executor_";
|
const std::string name2 = "disable_executor_";
|
||||||
const std::string name3 = "both_enable_";
|
|
||||||
const std::string name4 = "both_disable_";
|
|
||||||
|
|
||||||
const uint64_t id1 = 1;
|
const uint64_t id1 = 1;
|
||||||
const uint64_t id2 = 2;
|
const uint64_t id2 = 2;
|
||||||
const uint64_t id3 = 3;
|
|
||||||
const uint64_t id4 = 4;
|
|
||||||
|
|
||||||
ResourcePtr only_loader_ = nullptr;
|
ResourcePtr enable_executor_ = nullptr;
|
||||||
ResourcePtr only_executor_ = nullptr;
|
ResourcePtr disable_executor_ = nullptr;
|
||||||
ResourcePtr both_enable_ = nullptr;
|
|
||||||
ResourcePtr both_disable_ = nullptr;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, NAME) {
|
TEST_F(ResourceBaseTest, NAME) {
|
||||||
ASSERT_EQ(only_loader_->name(), name1);
|
ASSERT_EQ(enable_executor_->name(), name1);
|
||||||
ASSERT_EQ(only_executor_->name(), name2);
|
ASSERT_EQ(disable_executor_->name(), name2);
|
||||||
ASSERT_EQ(both_enable_->name(), name3);
|
|
||||||
ASSERT_EQ(both_disable_->name(), name4);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, TYPE) {
|
TEST_F(ResourceBaseTest, TYPE) {
|
||||||
ASSERT_EQ(only_loader_->type(), ResourceType::DISK);
|
ASSERT_EQ(enable_executor_->type(), ResourceType::CPU);
|
||||||
ASSERT_EQ(only_executor_->type(), ResourceType::CPU);
|
ASSERT_EQ(disable_executor_->type(), ResourceType::GPU);
|
||||||
ASSERT_EQ(both_enable_->type(), ResourceType::GPU);
|
|
||||||
ASSERT_EQ(both_disable_->type(), ResourceType::TEST);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, DEVICE_ID) {
|
TEST_F(ResourceBaseTest, DEVICE_ID) {
|
||||||
ASSERT_EQ(only_loader_->device_id(), id1);
|
ASSERT_EQ(enable_executor_->device_id(), id1);
|
||||||
ASSERT_EQ(only_executor_->device_id(), id2);
|
ASSERT_EQ(disable_executor_->device_id(), id2);
|
||||||
ASSERT_EQ(both_enable_->device_id(), id3);
|
|
||||||
ASSERT_EQ(both_disable_->device_id(), id4);
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, HAS_LOADER) {
|
|
||||||
ASSERT_TRUE(only_loader_->HasLoader());
|
|
||||||
ASSERT_FALSE(only_executor_->HasLoader());
|
|
||||||
ASSERT_TRUE(both_enable_->HasLoader());
|
|
||||||
ASSERT_FALSE(both_disable_->HasLoader());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, HAS_EXECUTOR) {
|
TEST_F(ResourceBaseTest, HAS_EXECUTOR) {
|
||||||
ASSERT_FALSE(only_loader_->HasExecutor());
|
ASSERT_TRUE(enable_executor_->HasExecutor());
|
||||||
ASSERT_TRUE(only_executor_->HasExecutor());
|
ASSERT_FALSE(disable_executor_->HasExecutor());
|
||||||
ASSERT_TRUE(both_enable_->HasExecutor());
|
|
||||||
ASSERT_FALSE(both_disable_->HasExecutor());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceBaseTest, DUMP) {
|
TEST_F(ResourceBaseTest, DUMP) {
|
||||||
ASSERT_FALSE(only_loader_->Dump().empty());
|
ASSERT_FALSE(enable_executor_->Dump().empty());
|
||||||
ASSERT_FALSE(only_executor_->Dump().empty());
|
ASSERT_FALSE(disable_executor_->Dump().empty());
|
||||||
ASSERT_FALSE(both_enable_->Dump().empty());
|
std::cout << *enable_executor_ << *disable_executor_;
|
||||||
ASSERT_FALSE(both_disable_->Dump().empty());
|
|
||||||
std::cout << *only_loader_ << *only_executor_ << *both_enable_ << *both_disable_;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/************ ResourceAdvanceTest ************/
|
/************ ResourceAdvanceTest ************/
|
||||||
|
@ -110,7 +85,7 @@ class ResourceAdvanceTest : public testing::Test {
|
||||||
disk_resource_ = ResourceFactory::Create("ssd", "DISK", 0);
|
disk_resource_ = ResourceFactory::Create("ssd", "DISK", 0);
|
||||||
cpu_resource_ = ResourceFactory::Create("cpu", "CPU", 0);
|
cpu_resource_ = ResourceFactory::Create("cpu", "CPU", 0);
|
||||||
gpu_resource_ = ResourceFactory::Create("gpu", "GPU", 0);
|
gpu_resource_ = ResourceFactory::Create("gpu", "GPU", 0);
|
||||||
test_resource_ = std::make_shared<TestResource>("test", 0, true, true);
|
test_resource_ = std::make_shared<TestResource>("test", 0, true);
|
||||||
resources_.push_back(disk_resource_);
|
resources_.push_back(disk_resource_);
|
||||||
resources_.push_back(cpu_resource_);
|
resources_.push_back(cpu_resource_);
|
||||||
resources_.push_back(gpu_resource_);
|
resources_.push_back(gpu_resource_);
|
||||||
|
|
|
@ -33,9 +33,9 @@ class ResourceMgrBaseTest : public testing::Test {
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
empty_mgr_ = std::make_shared<ResourceMgr>();
|
empty_mgr_ = std::make_shared<ResourceMgr>();
|
||||||
mgr1_ = std::make_shared<ResourceMgr>();
|
mgr1_ = std::make_shared<ResourceMgr>();
|
||||||
disk_res = std::make_shared<DiskResource>("disk", 0, true, false);
|
disk_res = std::make_shared<DiskResource>("disk", 0, false);
|
||||||
cpu_res = std::make_shared<CpuResource>("cpu", 1, true, false);
|
cpu_res = std::make_shared<CpuResource>("cpu", 1, false);
|
||||||
gpu_res = std::make_shared<GpuResource>("gpu", 2, true, true);
|
gpu_res = std::make_shared<GpuResource>("gpu", 2, true);
|
||||||
mgr1_->Add(ResourcePtr(disk_res));
|
mgr1_->Add(ResourcePtr(disk_res));
|
||||||
mgr1_->Add(ResourcePtr(cpu_res));
|
mgr1_->Add(ResourcePtr(cpu_res));
|
||||||
mgr1_->Add(ResourcePtr(gpu_res));
|
mgr1_->Add(ResourcePtr(gpu_res));
|
||||||
|
@ -53,20 +53,20 @@ class ResourceMgrBaseTest : public testing::Test {
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_F(ResourceMgrBaseTest, ADD) {
|
TEST_F(ResourceMgrBaseTest, ADD) {
|
||||||
auto resource = std::make_shared<TestResource>("test", 0, true, true);
|
auto resource = std::make_shared<TestResource>("test", 0, true);
|
||||||
auto ret = empty_mgr_->Add(ResourcePtr(resource));
|
auto ret = empty_mgr_->Add(ResourcePtr(resource));
|
||||||
ASSERT_EQ(ret.lock(), resource);
|
ASSERT_EQ(ret.lock(), resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
|
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
|
||||||
auto resource = std::make_shared<DiskResource>("disk", 0, true, true);
|
auto resource = std::make_shared<DiskResource>("disk", 0, true);
|
||||||
auto ret = empty_mgr_->Add(ResourcePtr(resource));
|
auto ret = empty_mgr_->Add(ResourcePtr(resource));
|
||||||
ASSERT_EQ(ret.lock(), resource);
|
ASSERT_EQ(ret.lock(), resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceMgrBaseTest, CONNECT) {
|
TEST_F(ResourceMgrBaseTest, CONNECT) {
|
||||||
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
|
auto resource1 = std::make_shared<TestResource>("resource1", 0, true);
|
||||||
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
|
auto resource2 = std::make_shared<TestResource>("resource2", 2, true);
|
||||||
empty_mgr_->Add(resource1);
|
empty_mgr_->Add(resource1);
|
||||||
empty_mgr_->Add(resource2);
|
empty_mgr_->Add(resource2);
|
||||||
Connection io("io", 500.0);
|
Connection io("io", 500.0);
|
||||||
|
@ -74,8 +74,8 @@ TEST_F(ResourceMgrBaseTest, CONNECT) {
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
|
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
|
||||||
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
|
auto resource1 = std::make_shared<TestResource>("resource1", 0, true);
|
||||||
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
|
auto resource2 = std::make_shared<TestResource>("resource2", 2, true);
|
||||||
empty_mgr_->Add(resource1);
|
empty_mgr_->Add(resource1);
|
||||||
empty_mgr_->Add(resource2);
|
empty_mgr_->Add(resource2);
|
||||||
Connection io("io", 500.0);
|
Connection io("io", 500.0);
|
||||||
|
@ -164,8 +164,8 @@ class ResourceMgrAdvanceTest : public testing::Test {
|
||||||
void
|
void
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
mgr1_ = std::make_shared<ResourceMgr>();
|
mgr1_ = std::make_shared<ResourceMgr>();
|
||||||
disk_res = std::make_shared<DiskResource>("disk", 0, true, false);
|
disk_res = std::make_shared<DiskResource>("disk", 0, false);
|
||||||
cpu_res = std::make_shared<CpuResource>("cpu", 0, true, true);
|
cpu_res = std::make_shared<CpuResource>("cpu", 0, true);
|
||||||
mgr1_->Add(ResourcePtr(disk_res));
|
mgr1_->Add(ResourcePtr(disk_res));
|
||||||
mgr1_->Add(ResourcePtr(cpu_res));
|
mgr1_->Add(ResourcePtr(cpu_res));
|
||||||
mgr1_->Start();
|
mgr1_->Start();
|
||||||
|
|
|
@ -37,10 +37,10 @@ class MockVecIndex : public engine::VecIndex {
|
||||||
const float* xt = nullptr) {
|
const float* xt = nullptr) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// engine::VecIndexPtr
|
// engine::VecIndexPtr
|
||||||
// Clone() override {
|
// Clone() override {
|
||||||
// return milvus::engine::VecIndexPtr();
|
// return milvus::engine::VecIndexPtr();
|
||||||
// }
|
// }
|
||||||
|
|
||||||
int64_t
|
int64_t
|
||||||
GetDeviceId() override {
|
GetDeviceId() override {
|
||||||
|
@ -99,8 +99,8 @@ class SchedulerTest : public testing::Test {
|
||||||
void
|
void
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
res_mgr_ = std::make_shared<ResourceMgr>();
|
res_mgr_ = std::make_shared<ResourceMgr>();
|
||||||
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false);
|
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, false);
|
||||||
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false);
|
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, false);
|
||||||
disk_resource_ = res_mgr_->Add(std::move(disk));
|
disk_resource_ = res_mgr_->Add(std::move(disk));
|
||||||
cpu_resource_ = res_mgr_->Add(std::move(cpu));
|
cpu_resource_ = res_mgr_->Add(std::move(cpu));
|
||||||
|
|
||||||
|
@ -143,10 +143,10 @@ class SchedulerTest2 : public testing::Test {
|
||||||
protected:
|
protected:
|
||||||
void
|
void
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false);
|
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, false);
|
||||||
ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0, true, false);
|
ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0, false);
|
||||||
ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1, true, false);
|
ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1, false);
|
||||||
ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2, true, false);
|
ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2, false);
|
||||||
|
|
||||||
res_mgr_ = std::make_shared<ResourceMgr>();
|
res_mgr_ = std::make_shared<ResourceMgr>();
|
||||||
disk_ = res_mgr_->Add(std::move(disk));
|
disk_ = res_mgr_->Add(std::move(disk));
|
||||||
|
@ -163,8 +163,8 @@ class SchedulerTest2 : public testing::Test {
|
||||||
res_mgr_->Connect("cpu0", "cpu2", IO);
|
res_mgr_->Connect("cpu0", "cpu2", IO);
|
||||||
|
|
||||||
#ifdef MILVUS_GPU_VERSION
|
#ifdef MILVUS_GPU_VERSION
|
||||||
ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0, true, true);
|
ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0);
|
||||||
ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1, true, true);
|
ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1);
|
||||||
gpu_0_ = res_mgr_->Add(std::move(gpu0));
|
gpu_0_ = res_mgr_->Add(std::move(gpu0));
|
||||||
gpu_1_ = res_mgr_->Add(std::move(gpu1));
|
gpu_1_ = res_mgr_->Add(std::move(gpu1));
|
||||||
res_mgr_->Connect("cpu1", "gpu0", PCIE1);
|
res_mgr_->Connect("cpu1", "gpu0", PCIE1);
|
||||||
|
|
|
@ -55,9 +55,9 @@ class RpcHandlerTest : public testing::Test {
|
||||||
SetUp() override {
|
SetUp() override {
|
||||||
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
|
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
|
||||||
res_mgr->Clear();
|
res_mgr->Clear();
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, false));
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0));
|
||||||
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
|
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0));
|
||||||
|
|
||||||
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
|
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
|
||||||
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
|
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
|
||||||
|
|
Loading…
Reference in New Issue