milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

clang format

pull/192/head
youny626 2019-11-05 13:35:24 +08:00
parent 29e8ef110f
commit bede77751e
70 changed files with 12797 additions and 13839 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
core/src/cache/Cache.inl vendored
View File

@ -15,24 +15,18 @@
// specific language governing permissions and limitations
// under the License.
namespace milvus {
namespace cache {
constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85;
template<typename ItemObj>
template <typename ItemObj>
Cache<ItemObj>::Cache(int64_t capacity, uint64_t cache_max_count)
: usage_(0),
capacity_(capacity),
freemem_percent_(DEFAULT_THRESHHOLD_PERCENT),
lru_(cache_max_count) {
// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
: usage_(0), capacity_(capacity), freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), lru_(cache_max_count) {
// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::set_capacity(int64_t capacity) {
if (capacity > 0) {
@ -41,23 +35,23 @@ Cache<ItemObj>::set_capacity(int64_t capacity) {
}
}
template<typename ItemObj>
template <typename ItemObj>
size_t
Cache<ItemObj>::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.size();
}
template<typename ItemObj>
template <typename ItemObj>
bool
Cache<ItemObj>::exists(const std::string &key) {
Cache<ItemObj>::exists(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.exists(key);
}
template<typename ItemObj>
template <typename ItemObj>
ItemObj
Cache<ItemObj>::get(const std::string &key) {
Cache<ItemObj>::get(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
if (!lru_.exists(key)) {
return nullptr;
@ -66,60 +60,59 @@ Cache<ItemObj>::get(const std::string &key) {
return lru_.get(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::insert(const std::string &key, const ItemObj &item) {
Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
if (item == nullptr) {
return;
}
// if(item->size() > capacity_) {
// SERVER_LOG_ERROR << "Item size " << item->size()
// << " is too large to insert into cache, capacity " << capacity_;
// return;
// }
// if(item->size() > capacity_) {
// SERVER_LOG_ERROR << "Item size " << item->size()
// << " is too large to insert into cache, capacity " << capacity_;
// return;
// }
//calculate usage
// calculate usage
{
std::lock_guard<std::mutex> lock(mutex_);
//if key already exist, subtract old item size
// if key already exist, subtract old item size
if (lru_.exists(key)) {
const ItemObj &old_item = lru_.get(key);
const ItemObj& old_item = lru_.get(key);
usage_ -= old_item->Size();
}
//plus new item size
// plus new item size
usage_ += item->Size();
}
//if usage exceed capacity, free some items
// if usage exceed capacity, free some items
if (usage_ > capacity_) {
SERVER_LOG_DEBUG << "Current usage " << usage_
<< " exceeds cache capacity " << capacity_
SERVER_LOG_DEBUG << "Current usage " << usage_ << " exceeds cache capacity " << capacity_
<< ", start free memory";
free_memory();
}
//insert new item
// insert new item
{
std::lock_guard<std::mutex> lock(mutex_);
lru_.put(key, item);
SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size()
<< " bytes into cache, usage: " << usage_ << " bytes";
SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size() << " bytes into cache, usage: " << usage_
<< " bytes";
}
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::erase(const std::string &key) {
Cache<ItemObj>::erase(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
if (!lru_.exists(key)) {
return;
}
const ItemObj &old_item = lru_.get(key);
const ItemObj& old_item = lru_.get(key);
usage_ -= old_item->Size();
SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size();
@ -127,7 +120,7 @@ Cache<ItemObj>::erase(const std::string &key) {
lru_.erase(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::clear() {
std::lock_guard<std::mutex> lock(mutex_);
@ -137,15 +130,16 @@ Cache<ItemObj>::clear() {
}
/* free memory space when CACHE occupation exceed its capacity */
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::free_memory() {
if (usage_ <= capacity_) return;
if (usage_ <= capacity_)
return;
int64_t threshhold = capacity_ * freemem_percent_;
int64_t delta_size = usage_ - threshhold;
if (delta_size <= 0) {
delta_size = 1;//ensure at least one item erased
delta_size = 1; // ensure at least one item erased
}
std::set<std::string> key_array;
@ -156,8 +150,8 @@ Cache<ItemObj>::free_memory() {
auto it = lru_.rbegin();
while (it != lru_.rend() && released_size < delta_size) {
auto &key = it->first;
auto &obj_ptr = it->second;
auto& key = it->first;
auto& obj_ptr = it->second;
key_array.emplace(key);
released_size += obj_ptr->Size();
@ -167,14 +161,14 @@ Cache<ItemObj>::free_memory() {
SERVER_LOG_DEBUG << "to be released memory size: " << released_size;
for (auto &key : key_array) {
for (auto& key : key_array) {
erase(key);
}
print();
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::print() {
size_t cache_count = 0;
@ -188,7 +182,5 @@ Cache<ItemObj>::print() {
SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes";
}
} // namespace cache
} // namespace milvus
} // namespace cache
} // namespace milvus

42
core/src/cache/CacheMgr.inl vendored
View File

@ -15,21 +15,18 @@
// specific language governing permissions and limitations
// under the License.
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
CacheMgr<ItemObj>::CacheMgr() {
}
template<typename ItemObj>
template <typename ItemObj>
CacheMgr<ItemObj>::~CacheMgr() {
}
template<typename ItemObj>
template <typename ItemObj>
uint64_t
CacheMgr<ItemObj>::ItemCount() const {
if (cache_ == nullptr) {
@ -37,12 +34,12 @@ CacheMgr<ItemObj>::ItemCount() const {
return 0;
}
return (uint64_t) (cache_->size());
return (uint64_t)(cache_->size());
}
template<typename ItemObj>
template <typename ItemObj>
bool
CacheMgr<ItemObj>::ItemExists(const std::string &key) {
CacheMgr<ItemObj>::ItemExists(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return false;
@ -51,9 +48,9 @@ CacheMgr<ItemObj>::ItemExists(const std::string &key) {
return cache_->exists(key);
}
template<typename ItemObj>
template <typename ItemObj>
ItemObj
CacheMgr<ItemObj>::GetItem(const std::string &key) {
CacheMgr<ItemObj>::GetItem(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return nullptr;
@ -62,9 +59,9 @@ CacheMgr<ItemObj>::GetItem(const std::string &key) {
return cache_->get(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
@ -74,9 +71,9 @@ CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::EraseItem(const std::string &key) {
CacheMgr<ItemObj>::EraseItem(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
@ -86,7 +83,7 @@ CacheMgr<ItemObj>::EraseItem(const std::string &key) {
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::PrintInfo() {
if (cache_ == nullptr) {
@ -97,7 +94,7 @@ CacheMgr<ItemObj>::PrintInfo() {
cache_->print();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::ClearCache() {
if (cache_ == nullptr) {
@ -108,7 +105,7 @@ CacheMgr<ItemObj>::ClearCache() {
cache_->clear();
}
template<typename ItemObj>
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheUsage() const {
if (cache_ == nullptr) {
@ -119,7 +116,7 @@ CacheMgr<ItemObj>::CacheUsage() const {
return cache_->usage();
}
template<typename ItemObj>
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheCapacity() const {
if (cache_ == nullptr) {
@ -130,7 +127,7 @@ CacheMgr<ItemObj>::CacheCapacity() const {
return cache_->capacity();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
if (cache_ == nullptr) {
@ -140,6 +137,5 @@ CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
cache_->set_capacity(capacity);
}
} // namespace cache
} // namespace milvus
} // namespace cache
} // namespace milvus

17
core/src/config.h.in
View File

@ -1,3 +1,20 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#cmakedefine MILVUS_VERSION "@MILVUS_VERSION@"
#cmakedefine BUILD_TYPE "@BUILD_TYPE@"
#cmakedefine BUILD_TIME @BUILD_TIME@

671
core/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

5107
core/src/external/easyloggingpp/easylogging++.cc vendored
View File

File diff suppressed because it is too large Load Diff

6756
core/src/external/easyloggingpp/easylogging++.h vendored
View File

File diff suppressed because it is too large Load Diff

10958
core/src/external/nlohmann/json.hpp vendored
View File

File diff suppressed because it is too large Load Diff

28
core/src/index/CMakeLists.txt
View File

@ -30,38 +30,38 @@ set(CMAKE_CXX_STANDARD 14)
set(KNOWHERE_VERSION_MAJOR "${knowhere_VERSION_MAJOR}")
set(KNOWHERE_VERSION_MINOR "${knowhere_VERSION_MINOR}")
set(KNOWHERE_VERSION_PATCH "${knowhere_VERSION_PATCH}")
if(KNOWHERE_VERSION_MAJOR STREQUAL ""
if (KNOWHERE_VERSION_MAJOR STREQUAL ""
OR KNOWHERE_VERSION_MINOR STREQUAL ""
OR KNOWHERE_VERSION_PATCH STREQUAL "")
message(FATAL_ERROR "Failed to determine Knowhere version from '${KNOWHERE_VERSION}'")
endif()
endif ()
message(STATUS "Knowhere version: "
"${KNOWHERE_VERSION_MAJOR}.${KNOWHERE_VERSION_MINOR}.${KNOWHERE_VERSION_PATCH} "
"(full: '${KNOWHERE_VERSION}')")
# if no build build type is specified, default to release builds
if(NOT CMAKE_BUILD_TYPE)
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif(NOT CMAKE_BUILD_TYPE)
endif (NOT CMAKE_BUILD_TYPE)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message(STATUS "building milvus_engine on x86 architecture")
set(KNOWHERE_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message(STATUS "building milvus_engine on ppc architecture")
set(KNOWHERE_BUILD_ARCH ppc64le)
else()
else ()
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(KNOWHERE_BUILD_ARCH unknown)
endif()
endif ()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
else()
else ()
set(BUILD_TYPE "debug")
endif()
endif ()
message(STATUS "Build type = ${BUILD_TYPE}")
set(INDEX_SOURCE_DIR ${PROJECT_SOURCE_DIR})
@ -103,12 +103,12 @@ add_subdirectory(knowhere)
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif()
endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
if(KNOWHERE_BUILD_TESTS)
if (KNOWHERE_BUILD_TESTS)
add_subdirectory(unittest)
endif()
endif ()
config_summary()

116
core/src/index/cmake/BuildUtilsCore.cmake
View File

@ -1,50 +1,50 @@
# Define a function that check last file modification
function(Check_Last_Modify cache_check_lists_file_path working_dir last_modified_commit_id)
if(EXISTS "${working_dir}")
if(EXISTS "${cache_check_lists_file_path}")
if (EXISTS "${working_dir}")
if (EXISTS "${cache_check_lists_file_path}")
set(GIT_LOG_SKIP_NUM 0)
set(_MATCH_ALL ON CACHE BOOL "Match all")
set(_LOOP_STATUS ON CACHE BOOL "Whether out of loop")
file(STRINGS ${cache_check_lists_file_path} CACHE_IGNORE_TXT)
while(_LOOP_STATUS)
foreach(_IGNORE_ENTRY ${CACHE_IGNORE_TXT})
if(NOT _IGNORE_ENTRY MATCHES "^[^#]+")
while (_LOOP_STATUS)
foreach (_IGNORE_ENTRY ${CACHE_IGNORE_TXT})
if (NOT _IGNORE_ENTRY MATCHES "^[^#]+")
continue()
endif()
endif ()
set(_MATCH_ALL OFF)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --name-status --pretty= WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE CHANGE_FILES)
if(NOT CHANGE_FILES STREQUAL "")
if (NOT CHANGE_FILES STREQUAL "")
string(REPLACE "\n" ";" _CHANGE_FILES ${CHANGE_FILES})
foreach(_FILE_ENTRY ${_CHANGE_FILES})
foreach (_FILE_ENTRY ${_CHANGE_FILES})
string(REGEX MATCH "[^ \t]+$" _FILE_NAME ${_FILE_ENTRY})
execute_process(COMMAND sh -c "echo ${_FILE_NAME} | grep ${_IGNORE_ENTRY}" RESULT_VARIABLE return_code)
if (return_code EQUAL 0)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
endforeach()
else()
endif ()
endforeach ()
else ()
set(_LOOP_STATUS OFF)
endif()
endforeach()
endif ()
endforeach ()
if(_MATCH_ALL)
if (_MATCH_ALL)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
endif ()
math(EXPR GIT_LOG_SKIP_NUM "${GIT_LOG_SKIP_NUM} + 1")
endwhile(_LOOP_STATUS)
else()
endwhile (_LOOP_STATUS)
else ()
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
endif()
else()
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
endif ()
else ()
message(FATAL_ERROR "The directory ${working_dir} does not exist")
endif()
endif ()
endfunction()
# Define a function that extracts a cached package
@ -83,15 +83,15 @@ endfunction()
# Define a function that to create a new cached package
function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path)
if(EXISTS ${package_file})
if (EXISTS ${package_file})
message(STATUS "Removing existing package file: ${package_file}")
file(REMOVE ${package_file})
endif()
endif ()
string(REGEX REPLACE "(.+)/.+$" "\\1" package_dir ${package_file})
if(NOT EXISTS ${package_dir})
if (NOT EXISTS ${package_dir})
file(MAKE_DIRECTORY ${package_dir})
endif()
endif ()
message(STATUS "Will create cached package file: ${package_file}")
@ -116,89 +116,89 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
if (ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
endif ()
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
if(NOT ARG_STATIC_LIB)
if (ARG_STATIC_LIB AND ARG_SHARED_LIB)
if (NOT ARG_STATIC_LIB)
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endif ()
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
endif ()
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
if (WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
else ()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
endif ()
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_STATIC_LIB)
endif ()
elseif (ARG_STATIC_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_SHARED_LIB)
endif ()
elseif (ARG_SHARED_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
if (WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
else ()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
endif ()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
if(ARG_INCLUDE_DIRECTORIES)
endif ()
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
else()
endif ()
else ()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endif ()
endfunction()

76
core/src/index/cmake/DefineOptionsCore.cmake
View File

@ -13,16 +13,16 @@ macro(define_option name description default)
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
if ("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
endif ()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
foreach (item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
endforeach ()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
@ -35,19 +35,19 @@ macro(define_option_string name description default)
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
if(MILVUS_CPU_VERSION)
if (MILVUS_CPU_VERSION)
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
else()
else ()
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
endif()
endif ()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
@ -55,11 +55,11 @@ set_option_category("Thirdparty")
set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO")
define_option_string(KNOWHERE_DEPENDENCY_SOURCE
"Method to use for acquiring KNOWHERE's build dependencies"
"${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
"Method to use for acquiring KNOWHERE's build dependencies"
"${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(KNOWHERE_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
@ -82,7 +82,7 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
if(MSVC)
if (MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
@ -90,16 +90,16 @@ if(MSVC)
OFF)
define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
endif()
endif ()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
if (BUILD_UNIT_TEST)
define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" ON)
else()
else ()
define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
endif (BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
@ -111,12 +111,12 @@ macro(config_summary)
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${INDEX_BINARY_DIR}/compile_commands.json")
endif()
endif ()
foreach(category ${KNOWHERE_OPTION_CATEGORIES})
foreach (category ${KNOWHERE_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
@ -124,50 +124,50 @@ macro(config_summary)
set(option_names ${KNOWHERE_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
foreach (name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
if (${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
endif ()
endforeach ()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
foreach (name ${option_names})
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
else ()
set(value "${${name}}")
endif()
endif ()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
if (${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
else ()
set(description_padding "
")
endif()
endif ()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
if ("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
endif ()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
endif ()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach ()
endforeach()
endforeach ()
endmacro()

174
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp
View File

@ -1,3 +1,20 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "IndexGPUIDMAP.h"
#include <faiss/AutoTune.h>
@ -18,96 +35,95 @@
namespace knowhere {
VectorIndexPtr
GPUIDMAP::CopyGpuToCpu(const Config &config) {
std::lock_guard<std::mutex> lk(mutex_);
VectorIndexPtr
GPUIDMAP::CopyGpuToCpu(const Config& config) {
std::lock_guard<std::mutex> lk(mutex_);
faiss::Index *device_index = index_.get();
faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
faiss::Index* device_index = index_.get();
faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index);
std::shared_ptr<faiss::Index> new_index;
new_index.reset(host_index);
return std::make_shared<IDMAP>(new_index);
std::shared_ptr<faiss::Index> new_index;
new_index.reset(host_index);
return std::make_shared<IDMAP>(new_index);
}
VectorIndexPtr
GPUIDMAP::Clone() {
auto cpu_idx = CopyGpuToCpu(Config());
if (auto idmap = std::dynamic_pointer_cast<IDMAP>(cpu_idx)) {
return idmap->CopyCpuToGpu(gpu_id_, Config());
} else {
KNOWHERE_THROW_MSG("IndexType not Support GpuClone");
}
}
VectorIndexPtr
GPUIDMAP::Clone() {
auto cpu_idx = CopyGpuToCpu(Config());
if (auto idmap = std::dynamic_pointer_cast<IDMAP>(cpu_idx)) {
return idmap->CopyCpuToGpu(gpu_id_, Config());
} else {
KNOWHERE_THROW_MSG("IndexType not Support GpuClone");
}
}
BinarySet
GPUIDMAP::SerializeImpl() {
try {
MemoryIOWriter writer;
{
faiss::Index *index = index_.get();
faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(index);
faiss::write_index(host_index, &writer);
delete host_index;
}
auto data = std::make_shared<uint8_t>();
data.reset(writer.data_);
BinarySet res_set;
res_set.Append("IVF", data, writer.rp);
return res_set;
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
}
void
GPUIDMAP::LoadImpl(const BinarySet &index_binary) {
auto binary = index_binary.GetByName("IVF");
MemoryIOReader reader;
BinarySet
GPUIDMAP::SerializeImpl() {
try {
MemoryIOWriter writer;
{
reader.total = binary->size;
reader.data_ = binary->data.get();
faiss::Index* index = index_.get();
faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index);
faiss::Index *index = faiss::read_index(&reader);
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) {
ResScope rs(res, gpu_id_, false);
auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index);
index_.reset(device_index);
res_ = res;
} else {
KNOWHERE_THROW_MSG("Load error, can't get gpu resource");
}
delete index;
faiss::write_index(host_index, &writer);
delete host_index;
}
}
auto data = std::make_shared<uint8_t>();
data.reset(writer.data_);
VectorIndexPtr
GPUIDMAP::CopyGpuToGpu(const int64_t &device_id, const Config &config) {
auto cpu_index = CopyGpuToCpu(config);
return std::static_pointer_cast<IDMAP>(cpu_index)->CopyCpuToGpu(device_id, config);
}
BinarySet res_set;
res_set.Append("IVF", data, writer.rp);
float *
GPUIDMAP::GetRawVectors() {
KNOWHERE_THROW_MSG("Not support");
return res_set;
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
int64_t *
GPUIDMAP::GetRawIds() {
KNOWHERE_THROW_MSG("Not support");
void
GPUIDMAP::LoadImpl(const BinarySet& index_binary) {
auto binary = index_binary.GetByName("IVF");
MemoryIOReader reader;
{
reader.total = binary->size;
reader.data_ = binary->data.get();
faiss::Index* index = faiss::read_index(&reader);
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) {
ResScope rs(res, gpu_id_, false);
auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index);
index_.reset(device_index);
res_ = res;
} else {
KNOWHERE_THROW_MSG("Load error, can't get gpu resource");
}
delete index;
}
}
void
GPUIDMAP::search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels,
const Config &cfg) {
ResScope rs(res_, gpu_id_);
index_->search(n, (float *) data, k, distances, labels);
}
VectorIndexPtr
GPUIDMAP::CopyGpuToGpu(const int64_t& device_id, const Config& config) {
auto cpu_index = CopyGpuToCpu(config);
return std::static_pointer_cast<IDMAP>(cpu_index)->CopyCpuToGpu(device_id, config);
}
} // knowhere
float*
GPUIDMAP::GetRawVectors() {
KNOWHERE_THROW_MSG("Not support");
}
int64_t*
GPUIDMAP::GetRawIds() {
KNOWHERE_THROW_MSG("Not support");
}
void
GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
ResScope rs(res_, gpu_id_);
index_->search(n, (float*)data, k, distances, labels);
}
} // namespace knowhere

70
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h
View File

@ -1,47 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "IndexGPUIVF.h"
#include "IndexIVF.h"
#include "IndexIDMAP.h"
#include "IndexIVF.h"
#include <memory>
#include <utility>
namespace knowhere {
class GPUIDMAP : public IDMAP, public GPUIndex {
public:
explicit GPUIDMAP(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &res)
: IDMAP(std::move(index)), GPUIndex(device_id, res) {
}
class GPUIDMAP : public IDMAP, public GPUIndex {
public:
explicit GPUIDMAP(std::shared_ptr<faiss::Index> index, const int64_t& device_id, ResPtr& res)
: IDMAP(std::move(index)), GPUIndex(device_id, res) {
}
VectorIndexPtr
CopyGpuToCpu(const Config &config) override;
VectorIndexPtr
CopyGpuToCpu(const Config& config) override;
float *
GetRawVectors() override;
float*
GetRawVectors() override;
int64_t *
GetRawIds() override;
int64_t*
GetRawIds() override;
VectorIndexPtr
Clone() override;
VectorIndexPtr
Clone() override;
VectorIndexPtr
CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
VectorIndexPtr
CopyGpuToGpu(const int64_t& device_id, const Config& config) override;
protected:
void
search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels,
const Config &cfg) override;
protected:
void
search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override;
BinarySet
SerializeImpl() override;
BinarySet
SerializeImpl() override;
void
LoadImpl(const BinarySet &index_binary) override;
};
void
LoadImpl(const BinarySet& index_binary) override;
};
using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>;
using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>;
} // knowhere
} // namespace knowhere

234
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
View File

@ -18,9 +18,9 @@
#include <faiss/IndexFlat.h>
#include <faiss/MetaIndexes.h>
#include <faiss/index_factory.h>
#include <faiss/clone_index.h>
#include <faiss/AutoTune.h>
#include <faiss/clone_index.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#ifdef MILVUS_GPU_VERSION
@ -38,165 +38,163 @@
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#endif
namespace knowhere {
BinarySet
IDMAP::Serialize() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
std::lock_guard<std::mutex> lk(mutex_);
return SerializeImpl();
BinarySet
IDMAP::Serialize() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
void
IDMAP::Load(const BinarySet &index_binary) {
std::lock_guard<std::mutex> lk(mutex_);
LoadImpl(index_binary);
std::lock_guard<std::mutex> lk(mutex_);
return SerializeImpl();
}
void
IDMAP::Load(const BinarySet& index_binary) {
std::lock_guard<std::mutex> lk(mutex_);
LoadImpl(index_binary);
}
DatasetPtr
IDMAP::Search(const DatasetPtr& dataset, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
DatasetPtr
IDMAP::Search(const DatasetPtr &dataset, const Config &config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
config->CheckValid();
// auto metric_type = config["metric_type"].as_string() == "L2" ?
// faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
// index_->metric_type = metric_type;
config->CheckValid();
// auto metric_type = config["metric_type"].as_string() == "L2" ?
// faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
// index_->metric_type = metric_type;
GETTENSOR(dataset)
GETTENSOR(dataset)
auto elems = rows * config->k;
auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems);
auto res_dis = (float*)malloc(sizeof(float) * elems);
auto elems = rows * config->k;
auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems);
auto res_dis = (float *) malloc(sizeof(float) * elems);
search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config());
search_impl(rows, (float *) p_data, config->k, res_dis, res_ids, Config());
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems);
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>(array, nullptr);
}
return std::make_shared<Dataset>(array, nullptr);
void
IDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
index_->search(n, (float*)data, k, distances, labels);
}
void
IDMAP::Add(const DatasetPtr& dataset, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
void
IDMAP::search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels, const Config &cfg) {
index_->search(n, (float *) data, k, distances, labels);
}
std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset)
void
IDMAP::Add(const DatasetPtr &dataset, const Config &config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
// TODO: magic here.
auto array = dataset->array()[0];
auto p_ids = array->data()->GetValues<int64_t>(1, 0);
std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset)
index_->add_with_ids(rows, (float*)p_data, p_ids);
}
// TODO: magic here.
auto array = dataset->array()[0];
auto p_ids = array->data()->GetValues<int64_t>(1, 0);
int64_t
IDMAP::Count() {
return index_->ntotal;
}
index_->add_with_ids(rows, (float *) p_data, p_ids);
}
int64_t
IDMAP::Count() {
return index_->ntotal;
}
int64_t
IDMAP::Dimension() {
return index_->d;
}
int64_t
IDMAP::Dimension() {
return index_->d;
}
// TODO(linxj): return const pointer
float *
IDMAP::GetRawVectors() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap *>(index_.get());
auto flat_index = dynamic_cast<faiss::IndexFlat *>(file_index->index);
return flat_index->xb.data();
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
float*
IDMAP::GetRawVectors() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap*>(index_.get());
auto flat_index = dynamic_cast<faiss::IndexFlat*>(file_index->index);
return flat_index->xb.data();
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
// TODO(linxj): return const pointer
int64_t *
IDMAP::GetRawIds() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap *>(index_.get());
return file_index->id_map.data();
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
int64_t*
IDMAP::GetRawIds() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap*>(index_.get());
return file_index->id_map.data();
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
const char *type = "IDMap,Flat";
const char* type = "IDMap,Flat";
void
IDMAP::Train(const Config &config) {
config->CheckValid();
void
IDMAP::Train(const Config& config) {
config->CheckValid();
auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type));
index_.reset(index);
}
auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type));
index_.reset(index);
}
VectorIndexPtr
IDMAP::Clone() {
std::lock_guard<std::mutex> lk(mutex_);
VectorIndexPtr
IDMAP::Clone() {
std::lock_guard<std::mutex> lk(mutex_);
auto clone_index = faiss::clone_index(index_.get());
std::shared_ptr<faiss::Index> new_index;
new_index.reset(clone_index);
return std::make_shared<IDMAP>(new_index);
}
VectorIndexPtr
IDMAP::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
auto clone_index = faiss::clone_index(index_.get());
std::shared_ptr<faiss::Index> new_index;
new_index.reset(clone_index);
return std::make_shared<IDMAP>(new_index);
}
VectorIndexPtr
IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
std::shared_ptr<faiss::Index> device_index;
device_index.reset(gpu_index);
return std::make_shared<GPUIDMAP>(device_index, device_id, res);
} else {
KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
}
std::shared_ptr<faiss::Index> device_index;
device_index.reset(gpu_index);
return std::make_shared<GPUIDMAP>(device_index, device_id, res);
} else {
KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
}
#else
KNOWHERE_THROW_MSG("Calling IDMAP::CopyCpuToGpu when we are using CPU version");
KNOWHERE_THROW_MSG("Calling IDMAP::CopyCpuToGpu when we are using CPU version");
#endif
}
}
void
IDMAP::Seal() {
// do nothing
}
void
IDMAP::Seal() {
// do nothing
}
} // namespace knowhere

5
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
View File

@ -21,9 +21,9 @@
#include <faiss/IndexIVF.h>
#include <faiss/IndexIVFFlat.h>
#include <faiss/IndexIVFPQ.h>
#include <faiss/index_io.h>
#include <faiss/index_factory.h>
#include <faiss/clone_index.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#ifdef MILVUS_GPU_VERSION
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/gpu/GpuCloner.h>
@ -240,7 +240,6 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6
VectorIndexPtr
IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {

1
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
View File

@ -62,7 +62,6 @@ IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index>& index) {
VectorIndexPtr
IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {

10
core/src/index/unittest/Helper.h
View File

@ -39,7 +39,7 @@ constexpr int64_t TEMPMEM = 1024 * 1024 * 300;
constexpr int64_t RESNUM = 2;
knowhere::IVFIndexPtr
IndexFactory(const std::string &type) {
IndexFactory(const std::string& type) {
if (type == "IVF") {
return std::make_shared<knowhere::IVF>();
} else if (type == "IVFPQ") {
@ -67,15 +67,15 @@ enum class ParameterType {
};
class ParamGenerator {
public:
static ParamGenerator &
public:
static ParamGenerator&
GetInstance() {
static ParamGenerator instance;
return instance;
}
knowhere::Config
Gen(const ParameterType &type) {
Gen(const ParameterType& type) {
if (type == ParameterType::ivf) {
auto tempconf = std::make_shared<knowhere::IVFCfg>();
tempconf->d = DIM;
@ -113,7 +113,7 @@ public:
#include <gtest/gtest.h>
class TestGpuIndexBase : public ::testing::Test {
protected:
protected:
void
SetUp() override {
#ifdef MILVUS_GPU_VERSION

2
core/src/index/unittest/test_idmap.cpp
View File

@ -22,8 +22,8 @@
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include "Helper.h"
#include "unittest/utils.h"

59
core/src/index/unittest/test_ivf.cpp
View File

@ -47,7 +47,7 @@ using ::testing::TestWithParam;
using ::testing::Values;
class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, ParameterType>> {
protected:
protected:
void
SetUp() override {
#ifdef MILVUS_GPU_VERSION
@ -80,16 +80,15 @@ protected:
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
Values(
#ifdef MILVUS_GPU_VERSION
std::make_tuple("GPUIVF", ParameterType::ivf),
std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
std::make_tuple("GPUIVF", ParameterType::ivf),
std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
#ifdef CUSTOMIZATION
std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
#endif
#endif
std::make_tuple("IVF", ParameterType::ivf),
std::make_tuple("IVFPQ", ParameterType::ivfpq),
std::make_tuple("IVFSQ", ParameterType::ivfsq)));
std::make_tuple("IVF", ParameterType::ivf), std::make_tuple("IVFPQ", ParameterType::ivfpq),
std::make_tuple("IVFSQ", ParameterType::ivfsq)));
TEST_P(IVFTest, ivf_basic) {
assert(!xb.empty());
@ -109,9 +108,9 @@ TEST_P(IVFTest, ivf_basic) {
}
TEST_P(IVFTest, ivf_serialize) {
auto serialize = [](const std::string &filename, knowhere::BinaryPtr &bin, uint8_t *ret) {
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void *>(bin->data.get()), bin->size);
writer(static_cast<void*>(bin->data.get()), bin->size);
FileIOReader reader(filename);
reader(ret, bin->size);
@ -216,18 +215,18 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone G <=> C [" << index_type << "] success" << std::endl;
});
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone G <=> C [" << index_type << "] success" << std::endl;
});
} else {
EXPECT_THROW(
{
std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
},
knowhere::KnowhereException);
{
std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
},
knowhere::KnowhereException);
}
}
@ -241,18 +240,18 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
} else {
EXPECT_THROW(
{
std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
},
knowhere::KnowhereException);
{
std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
},
knowhere::KnowhereException);
}
}
}

4
core/src/index/unittest/test_nsg/CMakeLists.txt
View File

@ -19,9 +19,9 @@ set(interface_src
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp
)
if(NOT TARGET test_nsg)
if (NOT TARGET test_nsg)
add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs})
endif()
endif ()
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
##############################

24
core/src/main.cpp
View File

@ -32,7 +32,7 @@
INITIALIZE_EASYLOGGINGPP
void
print_help(const std::string &app_name) {
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl;
std::cout << " Options:" << std::endl;
std::cout << " -h --help Print this help" << std::endl;
@ -61,15 +61,15 @@ print_banner() {
}
int
main(int argc, char *argv[]) {
main(int argc, char* argv[]) {
print_banner();
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
{"log_conf_file", required_argument, nullptr, 'l'},
{"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, nullptr, 'p'},
{nullptr, 0, nullptr, 0}};
{"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, nullptr, 'p'},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
int64_t start_daemonized = 0;
@ -78,7 +78,7 @@ main(int argc, char *argv[]) {
std::string pid_filename;
std::string app_name = argv[0];
milvus::server::Server &server = milvus::server::Server::GetInstance();
milvus::server::Server& server = milvus::server::Server::GetInstance();
milvus::Status s;
if (argc < 2) {
@ -90,21 +90,21 @@ main(int argc, char *argv[]) {
while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) {
switch (value) {
case 'c': {
char *config_filename_ptr = strdup(optarg);
char* config_filename_ptr = strdup(optarg);
config_filename = config_filename_ptr;
free(config_filename_ptr);
std::cout << "Loading configuration from: " << config_filename << std::endl;
break;
}
case 'l': {
char *log_filename_ptr = strdup(optarg);
char* log_filename_ptr = strdup(optarg);
log_config_file = log_filename_ptr;
free(log_filename_ptr);
std::cout << "Initializing log config from: " << log_config_file << std::endl;
break;
}
case 'p': {
char *pid_filename_ptr = strdup(optarg);
char* pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
free(pid_filename_ptr);
std::cout << pid_filename << std::endl;
@ -147,7 +147,7 @@ main(int argc, char *argv[]) {
return EXIT_SUCCESS;
FAIL:
FAIL:
std::cout << "Milvus server exit..." << std::endl;
return EXIT_FAILURE;
}

1
core/src/metrics/SystemInfo.cpp
View File

@ -214,7 +214,6 @@ SystemInfo::CPUPercent() {
std::vector<uint64_t>
SystemInfo::GPUMemoryTotal() {
// get GPU usage percent
if (!initialized_)
Init();

4
core/src/scheduler/ResourceMgr.cpp
View File

@ -79,7 +79,9 @@ ResourceMgr::Add(ResourcePtr&& resource) {
gpu_resources_.emplace_back(ResourceWPtr(resource));
break;
}
default: { break; }
default: {
break;
}
}
resources_.emplace_back(resource);

4
core/src/scheduler/Scheduler.cpp
View File

@ -123,7 +123,9 @@ Scheduler::OnLoadCompleted(const EventPtr& event) {
Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_, resource);
break;
}
default: { break; }
default: {
break;
}
}
resource->WakeupLoader();
}

4
core/src/scheduler/resource/Resource.cpp
View File

@ -44,7 +44,9 @@ ToString(ResourceType type) {
case ResourceType::GPU: {
return "GPU";
}
default: { return "UNKNOWN"; }
default: {
return "UNKNOWN";
}
}
}

5
core/src/server/Config.cpp
View File

@ -709,7 +709,7 @@ CheckResource(const std::string& value) {
int32_t gpu_index = std::stoi(s.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -1019,8 +1019,7 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
if (str != "cpu") {
value = std::stoi(str.substr(3));
}
else {
} else {
value = -1;
}

40
core/src/utils/BlockingQueue.inl
View File

@ -15,32 +15,26 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
namespace milvus {
namespace server {
template<typename T>
template <typename T>
void
BlockingQueue<T>::Put(const T &task) {
BlockingQueue<T>::Put(const T& task) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] {
return (queue_.size() < capacity_);
});
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
queue_.push(task);
empty_.notify_all();
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Take() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
queue_.pop();
@ -48,51 +42,45 @@ BlockingQueue<T>::Take() {
return front;
}
template<typename T>
template <typename T>
size_t
BlockingQueue<T>::Size() {
std::lock_guard<std::mutex> lock(mtx);
return queue_.size();
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Front() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
return front;
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Back() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T back(queue_.back());
return back;
}
template<typename T>
template <typename T>
bool
BlockingQueue<T>::Empty() {
std::unique_lock<std::mutex> lock(mtx);
return queue_.empty();
}
template<typename T>
template <typename T>
void
BlockingQueue<T>::SetCapacity(const size_t capacity) {
capacity_ = (capacity > 0 ? capacity : capacity_);
}
} // namespace server
} // namespace milvus
} // namespace server
} // namespace milvus

2
core/src/utils/ValidationUtil.cpp
View File

@ -170,7 +170,6 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc
Status
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
@ -192,7 +191,6 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
Status
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
#ifdef MILVUS_GPU_VERSION
cudaDeviceProp deviceProp;

1
core/src/wrapper/KnowhereResource.cpp
View File

@ -34,7 +34,6 @@ constexpr int64_t M_BYTE = 1024 * 1024;
Status
KnowhereResource::Initialize() {
#ifdef MILVUS_GPU_VERSION
struct GpuResourceSetting {

356
core/src/wrapper/VecImpl.cpp
View File

@ -24,8 +24,8 @@
#ifdef MILVUS_GPU_VERSION
#include <src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.h>
#include <src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h>
#include <src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.h>
#endif
@ -35,216 +35,214 @@
*/
namespace milvus {
namespace engine {
namespace engine {
Status
VecIndexImpl::BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg,
const int64_t &nt,
const float *xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
Status
VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
index_->set_index_model(model);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
index_->set_index_model(model);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
Status
VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) {
try {
auto k = cfg->k;
auto dataset = GenDataset(nq, dim, xq);
Status
VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg) {
try {
auto k = cfg->k;
auto dataset = GenDataset(nq, dim, xq);
Config search_cfg = cfg;
Config search_cfg = cfg;
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
// auto& dists = dis_array;
// std::stringstream ss_id;
// std::stringstream ss_dist;
// for (auto i = 0; i < 10; i++) {
// for (auto j = 0; j < k; ++j) {
// ss_id << *(ids->data()->GetValues<int64_t>(1, i * k + j)) << " ";
// ss_dist << *(dists->data()->GetValues<float>(1, i * k + j)) << " ";
// }
// ss_id << std::endl;
// ss_dist << std::endl;
// }
// std::cout << "id\n" << ss_id.str() << std::endl;
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
//{
// auto& ids = ids_array;
// auto& dists = dis_array;
// std::stringstream ss_id;
// std::stringstream ss_dist;
// for (auto i = 0; i < 10; i++) {
// for (auto j = 0; j < k; ++j) {
// ss_id << *(ids->data()->GetValues<int64_t>(1, i * k + j)) << " ";
// ss_dist << *(dists->data()->GetValues<float>(1, i * k + j)) << " ";
// }
// ss_id << std::endl;
// ss_dist << std::endl;
// }
// std::cout << "id\n" << ss_id.str() << std::endl;
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
memcpy(dist, p_dist, sizeof(float) * nq * k);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
// TODO(linxj): avoid copy here.
memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
memcpy(dist, p_dist, sizeof(float) * nq * k);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
knowhere::BinarySet
VecIndexImpl::Serialize() {
type = ConvertToCpuIndexType(type);
return index_->Serialize();
}
knowhere::BinarySet
VecIndexImpl::Serialize() {
type = ConvertToCpuIndexType(type);
return index_->Serialize();
}
Status
VecIndexImpl::Load(const knowhere::BinarySet &index_binary) {
index_->Load(index_binary);
dim = Dimension();
return Status::OK();
}
Status
VecIndexImpl::Load(const knowhere::BinarySet& index_binary) {
index_->Load(index_binary);
dim = Dimension();
return Status::OK();
}
int64_t
VecIndexImpl::Dimension() {
return index_->Dimension();
}
int64_t
VecIndexImpl::Dimension() {
return index_->Dimension();
}
int64_t
VecIndexImpl::Count() {
return index_->Count();
}
int64_t
VecIndexImpl::Count() {
return index_->Count();
}
IndexType
VecIndexImpl::GetType() {
return type;
}
IndexType
VecIndexImpl::GetType() {
return type;
}
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) {
// TODO(linxj): exception handle
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
new_index->dim = dim;
return new_index;
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
new_index->dim = dim;
return new_index;
#else
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToGpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToGpu when we are using CPU version");
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToGpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToGpu when we are using CPU version");
#endif
}
}
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config &cfg) {
// TODO(linxj): exception handle
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config& cfg) {
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
new_index->dim = dim;
return new_index;
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
new_index->dim = dim;
return new_index;
#else
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToCpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToCpu when we are using CPU version");
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToCpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToCpu when we are using CPU version");
#endif
}
}
VecIndexPtr
VecIndexImpl::Clone() {
// TODO(linxj): exception handle
auto clone_index = std::make_shared<VecIndexImpl>(index_->Clone(), type);
clone_index->dim = dim;
return clone_index;
}
VecIndexPtr
VecIndexImpl::Clone() {
// TODO(linxj): exception handle
auto clone_index = std::make_shared<VecIndexImpl>(index_->Clone(), type);
clone_index->dim = dim;
return clone_index;
}
int64_t
VecIndexImpl::GetDeviceId() {
int64_t
VecIndexImpl::GetDeviceId() {
#ifdef MILVUS_GPU_VERSION
if (auto device_idx = std::dynamic_pointer_cast<knowhere::GPUIndex>(index_)) {
return device_idx->GetGpuDevice();
}
if (auto device_idx = std::dynamic_pointer_cast<knowhere::GPUIndex>(index_)) {
return device_idx->GetGpuDevice();
}
#else
// else
return -1; // -1 == cpu
// else
return -1; // -1 == cpu
#endif
}
}
float *
BFIndex::GetRawVectors() {
auto raw_index = std::dynamic_pointer_cast<knowhere::IDMAP>(index_);
if (raw_index) {
return raw_index->GetRawVectors();
}
return nullptr;
}
float*
BFIndex::GetRawVectors() {
auto raw_index = std::dynamic_pointer_cast<knowhere::IDMAP>(index_);
if (raw_index) {
return raw_index->GetRawVectors();
}
return nullptr;
}
int64_t *
BFIndex::GetRawIds() {
return std::static_pointer_cast<knowhere::IDMAP>(index_)->GetRawIds();
}
int64_t*
BFIndex::GetRawIds() {
return std::static_pointer_cast<knowhere::IDMAP>(index_)->GetRawIds();
}
ErrorCode
BFIndex::Build(const Config &cfg) {
try {
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_UNEXPECTED_ERROR;
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_ERROR;
}
return KNOWHERE_SUCCESS;
}
ErrorCode
BFIndex::Build(const Config& cfg) {
try {
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_UNEXPECTED_ERROR;
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_ERROR;
}
return KNOWHERE_SUCCESS;
}
Status
BFIndex::BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg, const int64_t &nt,
const float *xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
Status
BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
} // namespace engine
} // namespace engine
} // namespace milvus

16
core/src/wrapper/VecIndex.cpp
View File

@ -28,12 +28,12 @@
#ifdef MILVUS_GPU_VERSION
#include <cuda.h>
#include "wrapper/gpu/GPUVecImpl.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "wrapper/gpu/GPUVecImpl.h"
#endif
namespace milvus {
@ -168,7 +168,9 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::NSG>(gpu_device);
break;
}
default: { return nullptr; }
default: {
return nullptr;
}
}
return std::make_shared<VecIndexImpl>(index, type);
}
@ -276,7 +278,9 @@ ConvertToCpuIndexType(const IndexType& type) {
case IndexType::FAISS_IVFSQ8_MIX: {
return IndexType::FAISS_IVFSQ8_CPU;
}
default: { return type; }
default: {
return type;
}
}
}
@ -291,7 +295,9 @@ ConvertToGpuIndexType(const IndexType& type) {
case IndexType::FAISS_IVFSQ8_CPU: {
return IndexType::FAISS_IVFSQ8_GPU;
}
default: { return type; }
default: {
return type;
}
}
}

6
core/src/wrapper/WrapperException.cpp
View File

@ -20,9 +20,11 @@
namespace milvus {
namespace engine {
WrapperException::WrapperException(const std::string &msg) : msg(msg) {}
WrapperException::WrapperException(const std::string& msg) : msg(msg) {
}
const char *WrapperException::what() const noexcept {
const char*
WrapperException::what() const noexcept {
return msg.c_str();
}

3
core/src/wrapper/WrapperException.h
View File

@ -27,7 +27,8 @@ class WrapperException : public std::exception {
public:
explicit WrapperException(const std::string& msg);
const char* what() const noexcept override;
const char*
what() const noexcept override;
const std::string msg;
};

9
core/src/wrapper/gpu/GPUVecImpl.cpp
View File

@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "wrapper/VecImpl.h"
#include "GPUVecImpl.h"
#include "src/wrapper/DataTransfer.h"
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "src/wrapper/DataTransfer.h"
#include "utils/Log.h"
#include "wrapper/VecImpl.h"
/*
* no parameter check in this layer.
@ -34,7 +34,6 @@
namespace milvus {
namespace engine {
// TODO(linxj): add lock here.
Status
IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,

29
core/src/wrapper/gpu/GPUVecImpl.h
View File

@ -20,46 +20,47 @@
#include <memory>
#include <utility>
#include "wrapper/VecIndex.h"
#include "knowhere/index/vector_index/VectorIndex.h"
#include "wrapper/VecImpl.h"
#include "wrapper/VecIndex.h"
namespace milvus {
namespace engine {
class IVFMixIndex : public VecIndexImpl {
public:
explicit IVFMixIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType &type)
: VecIndexImpl(std::move(index), type) {
public:
explicit IVFMixIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType& type)
: VecIndexImpl(std::move(index), type) {
}
Status
BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg, const int64_t &nt,
const float *xt) override;
BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) override;
Status
Load(const knowhere::BinarySet &index_binary) override;
Load(const knowhere::BinarySet& index_binary) override;
};
class IVFHybridIndex : public IVFMixIndex {
public:
explicit IVFHybridIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType &type)
: IVFMixIndex(std::move(index), type) {
public:
explicit IVFHybridIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType& type)
: IVFMixIndex(std::move(index), type) {
}
knowhere::QuantizerPtr
LoadQuantizer(const Config &conf) override;
LoadQuantizer(const Config& conf) override;
Status
SetQuantizer(const knowhere::QuantizerPtr &q) override;
SetQuantizer(const knowhere::QuantizerPtr& q) override;
Status
UnsetQuantizer() override;
std::pair<VecIndexPtr, knowhere::QuantizerPtr>
CopyToGpuWithQuantizer(const int64_t &device_id, const Config &cfg) override;
CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) override;
VecIndexPtr
LoadData(const knowhere::QuantizerPtr &q, const Config &conf) override;
LoadData(const knowhere::QuantizerPtr& q, const Config& conf) override;
};
} // namespace engine

6
core/unittest/CMakeLists.txt
View File

@ -17,9 +17,9 @@
# under the License.
#-------------------------------------------------------------------------------
foreach(dir ${INDEX_INCLUDE_DIRS})
foreach (dir ${INDEX_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
endforeach ()
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
@ -127,7 +127,7 @@ if (MILVUS_GPU_VERSION)
set(common_files ${common_files}
${wrapper_gpu_files}
)
endif()
endif ()
add_subdirectory(db)
add_subdirectory(wrapper)

75
core/unittest/db/test_db.cpp
View File

@ -15,25 +15,24 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include "db/DBFactory.h"
#include "cache/CpuCacheMgr.h"
#include "utils/CommonUtil.h"
#include "db/Constants.h"
#include "db/DB.h"
#include "db/DBFactory.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
#include "db/utils.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
#include <thread>
namespace {
static const char *TABLE_NAME = "test_group";
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
@ -49,10 +48,10 @@ BuildTableSchema() {
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
@ -68,16 +67,15 @@ CurrentTmDate(int64_t offset_day = 0) {
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
std::string str =
std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday);
return str;
}
void
ConvertTimeRangeToDBDates(const std::string &start_value,
const std::string &end_value,
std::vector<milvus::engine::meta::DateT> &dates) {
ConvertTimeRangeToDBDates(const std::string& start_value, const std::string& end_value,
std::vector<milvus::engine::meta::DateT>& dates) {
dates.clear();
time_t tt_start, tt_end;
@ -90,8 +88,7 @@ ConvertTimeRangeToDBDates(const std::string &start_value,
return;
}
int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) /
DAY_SECONDS;
int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / DAY_SECONDS;
if (days == 0) {
return;
}
@ -101,13 +98,12 @@ ConvertTimeRangeToDBDates(const std::string &start_value,
tm tm_day;
milvus::server::CommonUtil::ConvertTime(tt_day, tm_day);
int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 +
tm_day.tm_mday;//according to db logic
int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic
dates.push_back(date);
}
}
} // namespace
} // namespace
TEST_F(DBTest, CONFIG_TEST) {
{
@ -232,7 +228,7 @@ TEST_F(DBTest, DB_TEST) {
TEST_F(DBTest, SEARCH_TEST) {
std::string config_path(CONFIG_PATH);
config_path += CONFIG_FILE;
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(config_path);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
@ -266,22 +262,24 @@ TEST_F(DBTest, SEARCH_TEST) {
}
// result data
//std::vector<long> nns_gt(k*nq);
// std::vector<long> nns_gt(k*nq);
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
// std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
if (j == 200) {
sleep(1);
}
ASSERT_TRUE(stat.ok());
}
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
milvus::engine::QueryResults results;
@ -289,7 +287,7 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
{ // search by specify index file
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
milvus::engine::QueryResults results;
@ -298,9 +296,9 @@ TEST_F(DBTest, SEARCH_TEST) {
}
#ifdef CUSTOMIZATION
//test FAISS_IVFSQ8H optimizer
// test FAISS_IVFSQ8H optimizer
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
milvus::engine::QueryResults results;
@ -314,7 +312,7 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
{ // search by specify index file
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
milvus::engine::QueryResults results;
@ -322,7 +320,6 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
#endif
}
@ -348,8 +345,8 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
}
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
int64_t prev_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);
@ -415,12 +412,12 @@ TEST_F(DBTest, INDEX_TEST) {
ASSERT_EQ(vector_ids.size(), nb);
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int) milvus::engine::MetricType::IP;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int)milvus::engine::MetricType::IP;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFFLAT;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
@ -449,7 +446,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for (auto &schema : table_schema_array) {
for (auto& schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;

67
core/unittest/db/test_db_mysql.cpp
View File

@ -15,21 +15,21 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/Constants.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include "db/utils.h"
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
#include <thread>
namespace {
static const char *TABLE_NAME = "test_group";
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
@ -39,22 +39,22 @@ BuildTableSchema() {
milvus::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
table_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
} // namespace
} // namespace
TEST_F(MySqlDBTest, DB_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
@ -102,10 +102,10 @@ TEST_F(MySqlDBTest, DB_TEST) {
ASSERT_TRUE(stat.ok());
for (auto k = 0; k < qb; ++k) {
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
bool exists = false;
for (auto &result : results[k]) {
for (auto& result : results[k]) {
if (result.first == target_ids[k]) {
exists = true;
}
@ -128,12 +128,12 @@ TEST_F(MySqlDBTest, DB_TEST) {
int loop = INSERT_LOOP;
for (auto i = 0; i < loop; ++i) {
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
// } else {
// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
// }
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
// } else {
// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
// }
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
@ -173,20 +173,22 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
}
// result data
//std::vector<long> nns_gt(k*nq);
// std::vector<long> nns_gt(k*nq);
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
// std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
if (j == 200) {
sleep(1);
}
ASSERT_TRUE(stat.ok());
}
sleep(2); // wait until build index finish
sleep(2); // wait until build index finish
milvus::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
@ -201,7 +203,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for (auto &schema : table_schema_array) {
for (auto& schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
@ -241,7 +243,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
TEST_F(MySqlDBTest, DELETE_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl;
// std::cout << stat.ToString() << std::endl;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
@ -267,14 +269,13 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
// std::vector<engine::meta::DateT> dates;
// stat = db_->DeleteTable(TABLE_NAME, dates);
//// std::cout << "5 sec start" << std::endl;
// std::this_thread::sleep_for(std::chrono::seconds(5));
//// std::cout << "5 sec finish" << std::endl;
// ASSERT_TRUE(stat.ok());
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
// std::vector<engine::meta::DateT> dates;
// stat = db_->DeleteTable(TABLE_NAME, dates);
//// std::cout << "5 sec start" << std::endl;
// std::this_thread::sleep_for(std::chrono::seconds(5));
//// std::cout << "5 sec finish" << std::endl;
// ASSERT_TRUE(stat.ok());
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
}

59
core/unittest/db/test_engine.cpp
View File

@ -26,55 +26,36 @@
TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::INVALID,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::INVALID, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr == nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IDMAP,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IDMAP, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr =
milvus::engine::EngineFactory::Build(512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFSQ8, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IVFSQ8,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::NSG_MIX,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::NSG_MIX, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
@ -84,21 +65,17 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension,
file_path,
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
std::vector<float> data;
std::vector<int64_t> ids;
const int row_count = 10000;
data.reserve(row_count*dimension);
data.reserve(row_count * dimension);
ids.reserve(row_count);
for (int64_t i = 0; i < row_count; i++) {
ids.push_back(i);
for (uint16_t k = 0; k < dimension; k++) {
data.push_back(i*dimension + k);
data.push_back(i * dimension + k);
}
}
@ -109,14 +86,14 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Count(), ids.size());
status = engine_ptr->CopyToGpu(0, false);
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
auto new_engine = engine_ptr->Clone();
ASSERT_EQ(new_engine->Dimension(), dimension);
ASSERT_EQ(new_engine->Count(), ids.size());
status = new_engine->CopyToCpu();
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
}

54
core/unittest/db/test_mem.cpp
View File

@ -15,25 +15,24 @@
// specific language governing permissions and limitations
// under the License.
#include "gtest/gtest.h"
#include "db/insert/VectorSource.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/MemTable.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "db/insert/MemTable.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/VectorSource.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "db/utils.h"
#include "metrics/Metrics.h"
#include <boost/filesystem.hpp>
#include <thread>
#include <chrono>
#include <cmath>
#include <fstream>
#include <iostream>
#include <cmath>
#include <random>
#include <chrono>
#include <thread>
namespace {
@ -42,8 +41,7 @@ static constexpr int64_t TABLE_DIM = 256;
std::string
GetTableName() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
static std::string table_name = std::to_string(micros);
return table_name;
}
@ -63,11 +61,10 @@ BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.resize(n * TABLE_DIM);
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++)
data[TABLE_DIM * i + j] = drand48();
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
}
}
} // namespace
} // namespace
TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
@ -86,12 +83,10 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
milvus::engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
milvus::engine::ExecutionEnginePtr execution_engine_ =
milvus::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(milvus::engine::EngineType)table_file_schema.engine_type_,
(milvus::engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
milvus::engine::ExecutionEnginePtr execution_engine_ = milvus::engine::EngineFactory::Build(
table_file_schema.dimension_, table_file_schema.location_,
(milvus::engine::EngineType)table_file_schema.engine_type_,
(milvus::engine::MetricType)table_file_schema.metric_type_, table_schema.nlist_);
milvus::engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
@ -129,7 +124,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
status = mem_table_file.Add(source, vector_ids);
ASSERT_TRUE(status.ok());
// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl;
// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl;
vector_ids = source->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
@ -141,8 +136,8 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
@ -163,8 +158,8 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
milvus::engine::VectorSourcePtr
source_100 = std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::VectorSourcePtr source_100 =
std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::MemTable mem_table(GetTableName(), impl_, options);
@ -184,8 +179,8 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
@ -239,7 +234,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk
std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
std::random_device rd;
std::mt19937 gen(rd());
@ -400,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[0], nb);
ASSERT_TRUE(stat.ok());
nb = 262144; //512M
nb = 262144; // 512M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
@ -412,7 +407,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[0], nb / 2);
ASSERT_TRUE(stat.ok());
nb = 65536; //128M
nb = 65536; // 128M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
@ -432,4 +427,3 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[i], i + nb);
}
}

35
core/unittest/db/test_meta.cpp
View File

@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/Constants.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/utils.h"
#include <gtest/gtest.h>
#include <thread>
#include <stdlib.h>
#include <time.h>
#include <thread>
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
@ -155,7 +155,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
@ -200,7 +200,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
@ -293,16 +293,13 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
@ -315,19 +312,15 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);

47
core/unittest/db/test_meta_mysql.cpp
View File

@ -15,17 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/utils.h"
#include <iostream>
#include <thread>
#include <stdlib.h>
#include <time.h>
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
#include <stdlib.h>
#include <time.h>
#include <iostream>
#include <thread>
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
@ -52,7 +52,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = "";
status = impl_->CreateTable(table);
// ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
status = impl_->DropAll();
ASSERT_TRUE(status.ok());
@ -79,8 +79,8 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL);
auto file_id = table_file.file_id_;
@ -159,7 +159,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
@ -167,7 +167,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
}
std::vector<int> file_types = {
(int) milvus::engine::meta::TableFileSchema::NEW,
(int)milvus::engine::meta::TableFileSchema::NEW,
};
std::vector<std::string> file_ids;
status = impl.FilesByType(table_id, file_types, file_ids);
@ -219,7 +219,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
@ -313,16 +313,13 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
@ -335,19 +332,15 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);

31
core/unittest/db/test_misc.cpp
View File

@ -16,15 +16,15 @@
// under the License.
#include "db/Options.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/engine/EngineFactory.h"
#include "db/Utils.h"
#include "utils/Status.h"
#include "db/engine/EngineFactory.h"
#include "db/meta/SqliteMetaImpl.h"
#include "utils/Exception.h"
#include "utils/Status.h"
#include <gtest/gtest.h>
#include <thread>
#include <boost/filesystem.hpp>
#include <thread>
#include <vector>
TEST(DBMiscTest, EXCEPTION_TEST) {
@ -40,7 +40,7 @@ TEST(DBMiscTest, EXCEPTION_TEST) {
TEST(DBMiscTest, OPTIONS_TEST) {
try {
milvus::engine::ArchiveConf archive("$$##");
} catch (std::exception &ex) {
} catch (std::exception& ex) {
ASSERT_TRUE(true);
}
@ -61,10 +61,7 @@ TEST(DBMiscTest, OPTIONS_TEST) {
{
milvus::engine::ArchiveConf archive("delete");
milvus::engine::ArchiveConf::CriteriaT criterial = {
{"disk", 1024},
{"days", 100}
};
milvus::engine::ArchiveConf::CriteriaT criterial = {{"disk", 1024}, {"days", 100}};
archive.SetCriterias(criterial);
auto crit = archive.GetCriterias();
@ -95,17 +92,17 @@ TEST(DBMiscTest, UTILS_TEST) {
auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
}
// options.slave_paths.push_back("/");
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
//
// options.path = "/";
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
// options.slave_paths.push_back("/");
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
//
// options.path = "/";
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
milvus::engine::meta::TableFileSchema file;
file.id_ = 50;

278
core/unittest/db/test_search.cpp
View File

@ -20,32 +20,28 @@
#include <vector>
#include "scheduler/task/SearchTask.h"
#include "utils/TimeRecorder.h"
#include "utils/ThreadPool.h"
#include "utils/TimeRecorder.h"
namespace {
namespace ms = milvus::scheduler;
void
BuildResult(std::vector<int64_t>& output_ids,
std::vector<float>& output_distance,
uint64_t input_k,
uint64_t topk,
uint64_t nq,
bool ascending) {
BuildResult(std::vector<int64_t>& output_ids, std::vector<float>& output_distance, uint64_t input_k, uint64_t topk,
uint64_t nq, bool ascending) {
output_ids.clear();
output_ids.resize(nq * topk);
output_distance.clear();
output_distance.resize(nq * topk);
for (uint64_t i = 0; i < nq; i++) {
//insert valid items
// insert valid items
for (uint64_t j = 0; j < input_k; j++) {
output_ids[i * topk + j] = (int64_t)(drand48() * 100000);
output_distance[i * topk + j] = ascending ? (j + drand48()) : ((input_k - j) + drand48());
}
//insert invalid items
// insert invalid items
for (uint64_t j = input_k; j < topk; j++) {
output_ids[i * topk + j] = -1;
output_distance[i * topk + j] = -1.0;
@ -54,13 +50,8 @@ BuildResult(std::vector<int64_t>& output_ids,
}
void
CopyResult(std::vector<int64_t>& output_ids,
std::vector<float>& output_distance,
uint64_t output_topk,
std::vector<int64_t>& input_ids,
std::vector<float>& input_distance,
uint64_t input_topk,
uint64_t nq) {
CopyResult(std::vector<int64_t>& output_ids, std::vector<float>& output_distance, uint64_t output_topk,
std::vector<int64_t>& input_ids, std::vector<float>& input_distance, uint64_t input_topk, uint64_t nq) {
ASSERT_TRUE(input_ids.size() >= nq * input_topk);
ASSERT_TRUE(input_distance.size() >= nq * input_topk);
ASSERT_TRUE(output_topk <= input_topk);
@ -78,31 +69,23 @@ CopyResult(std::vector<int64_t>& output_ids,
}
void
CheckTopkResult(const std::vector<int64_t>& input_ids_1,
const std::vector<float>& input_distance_1,
const std::vector<int64_t>& input_ids_2,
const std::vector<float>& input_distance_2,
uint64_t topk,
uint64_t nq,
bool ascending,
const milvus::scheduler::ResultSet& result) {
CheckTopkResult(const std::vector<int64_t>& input_ids_1, const std::vector<float>& input_distance_1,
const std::vector<int64_t>& input_ids_2, const std::vector<float>& input_distance_2, uint64_t topk,
uint64_t nq, bool ascending, const milvus::scheduler::ResultSet& result) {
ASSERT_EQ(result.size(), nq);
ASSERT_EQ(input_ids_1.size(), input_distance_1.size());
ASSERT_EQ(input_ids_2.size(), input_distance_2.size());
for (int64_t i = 0; i < nq; i++) {
std::vector<float>
src_vec(input_distance_1.begin() + i * topk, input_distance_1.begin() + (i + 1) * topk);
src_vec.insert(src_vec.end(),
input_distance_2.begin() + i * topk,
input_distance_2.begin() + (i + 1) * topk);
std::vector<float> src_vec(input_distance_1.begin() + i * topk, input_distance_1.begin() + (i + 1) * topk);
src_vec.insert(src_vec.end(), input_distance_2.begin() + i * topk, input_distance_2.begin() + (i + 1) * topk);
if (ascending) {
std::sort(src_vec.begin(), src_vec.end());
} else {
std::sort(src_vec.begin(), src_vec.end(), std::greater<float>());
}
//erase invalid items
// erase invalid items
std::vector<float>::iterator iter;
for (iter = src_vec.begin(); iter != src_vec.end();) {
if (*iter < 0.0)
@ -124,7 +107,7 @@ CheckTopkResult(const std::vector<int64_t>& input_ids_1,
}
}
} // namespace
} // namespace
void
MergeTopkToResultSetTest(uint64_t topk_1, uint64_t topk_2, uint64_t nq, uint64_t topk, bool ascending) {
@ -159,7 +142,7 @@ TEST(DBSearchTest, MERGE_RESULT_SET_TEST) {
MergeTopkToResultSetTest(TOP_K / 2, TOP_K / 3, NQ, TOP_K, false);
}
//void MergeTopkArrayTest(uint64_t topk_1, uint64_t topk_2, uint64_t nq, uint64_t topk, bool ascending) {
// void MergeTopkArrayTest(uint64_t topk_1, uint64_t topk_2, uint64_t nq, uint64_t topk, bool ascending) {
// std::vector<int64_t> ids1, ids2;
// std::vector<float> dist1, dist2;
// ms::ResultSet result;
@ -191,7 +174,7 @@ TEST(DBSearchTest, MERGE_RESULT_SET_TEST) {
// }
//}
//TEST(DBSearchTest, MERGE_ARRAY_TEST) {
// TEST(DBSearchTest, MERGE_ARRAY_TEST) {
// uint64_t NQ = 15;
// uint64_t TOP_K = 64;
//
@ -219,7 +202,7 @@ TEST(DBSearchTest, MERGE_RESULT_SET_TEST) {
//}
TEST(DBSearchTest, REDUCE_PERF_TEST) {
int32_t index_file_num = 478; /* sift1B dataset, index files num */
int32_t index_file_num = 478; /* sift1B dataset, index files num */
bool ascending = true;
std::vector<int32_t> thread_vec = {4, 8};
@ -255,133 +238,128 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
CopyResult(id_vec_1[i], dist_vec_1[i], top_k, id_vec[i], dist_vec[i], TOPK, nq);
}
std::string str1 = "Method-1 " + std::to_string(max_thread_num) + " " +
std::to_string(nq) + " " + std::to_string(top_k);
std::string str1 = "Method-1 " + std::to_string(max_thread_num) + " " + std::to_string(nq) + " " +
std::to_string(top_k);
milvus::TimeRecorder rc1(str1);
///////////////////////////////////////////////////////////////////////////////////////
/* method-1 */
for (i = 0; i < index_file_num; i++) {
ms::XSearchTask::MergeTopkToResultSet(id_vec_1[i],
dist_vec_1[i],
top_k,
nq,
top_k,
ascending,
ms::XSearchTask::MergeTopkToResultSet(id_vec_1[i], dist_vec_1[i], top_k, nq, top_k, ascending,
final_result);
ASSERT_EQ(final_result.size(), nq);
}
rc1.RecordSection("reduce done");
// ///////////////////////////////////////////////////////////////////////////////////////
// /* method-2 */
// std::vector<std::vector<int64_t>> id_vec_2(index_file_num);
// std::vector<std::vector<float>> dist_vec_2(index_file_num);
// std::vector<uint64_t> k_vec_2(index_file_num);
// for (i = 0; i < index_file_num; i++) {
// CopyResult(id_vec_2[i], dist_vec_2[i], top_k, id_vec[i], dist_vec[i], TOPK, nq);
// k_vec_2[i] = top_k;
// }
//
// std::string str2 = "Method-2 " + std::to_string(max_thread_num) + " " +
// std::to_string(nq) + " " + std::to_string(top_k);
// milvus::TimeRecorder rc2(str2);
//
// for (step = 1; step < index_file_num; step *= 2) {
// for (i = 0; i + step < index_file_num; i += step * 2) {
// ms::XSearchTask::MergeTopkArray(id_vec_2[i], dist_vec_2[i], k_vec_2[i],
// id_vec_2[i + step], dist_vec_2[i + step], k_vec_2[i + step],
// nq, top_k, ascending);
// }
// }
// ms::XSearchTask::MergeTopkToResultSet(id_vec_2[0],
// dist_vec_2[0],
// k_vec_2[0],
// nq,
// top_k,
// ascending,
// final_result_2);
// ASSERT_EQ(final_result_2.size(), nq);
//
// rc2.RecordSection("reduce done");
//
// for (i = 0; i < nq; i++) {
// ASSERT_EQ(final_result[i].size(), final_result_2[i].size());
// for (k = 0; k < final_result[i].size(); k++) {
// if (final_result[i][k].first != final_result_2[i][k].first) {
// std::cout << i << " " << k << std::endl;
// }
// ASSERT_EQ(final_result[i][k].first, final_result_2[i][k].first);
// ASSERT_EQ(final_result[i][k].second, final_result_2[i][k].second);
// }
// }
//
// ///////////////////////////////////////////////////////////////////////////////////////
// /* method-3 parallel */
// std::vector<std::vector<int64_t>> id_vec_3(index_file_num);
// std::vector<std::vector<float>> dist_vec_3(index_file_num);
// std::vector<uint64_t> k_vec_3(index_file_num);
// for (i = 0; i < index_file_num; i++) {
// CopyResult(id_vec_3[i], dist_vec_3[i], top_k, id_vec[i], dist_vec[i], TOPK, nq);
// k_vec_3[i] = top_k;
// }
//
// std::string str3 = "Method-3 " + std::to_string(max_thread_num) + " " +
// std::to_string(nq) + " " + std::to_string(top_k);
// milvus::TimeRecorder rc3(str3);
//
// for (step = 1; step < index_file_num; step *= 2) {
// for (i = 0; i + step < index_file_num; i += step * 2) {
// threads_list.push_back(
// threadPool.enqueue(ms::XSearchTask::MergeTopkArray,
// std::ref(id_vec_3[i]),
// std::ref(dist_vec_3[i]),
// std::ref(k_vec_3[i]),
// std::ref(id_vec_3[i + step]),
// std::ref(dist_vec_3[i + step]),
// std::ref(k_vec_3[i + step]),
// nq,
// top_k,
// ascending));
// }
//
// while (threads_list.size() > 0) {
// int nready = 0;
// for (auto it = threads_list.begin(); it != threads_list.end(); it = it) {
// auto &p = *it;
// std::chrono::milliseconds span(0);
// if (p.wait_for(span) == std::future_status::ready) {
// threads_list.erase(it++);
// ++nready;
// } else {
// ++it;
// }
// }
//
// if (nready == 0) {
// std::this_thread::yield();
// }
// }
// }
// ms::XSearchTask::MergeTopkToResultSet(id_vec_3[0],
// dist_vec_3[0],
// k_vec_3[0],
// nq,
// top_k,
// ascending,
// final_result_3);
// ASSERT_EQ(final_result_3.size(), nq);
//
// rc3.RecordSection("reduce done");
//
// for (i = 0; i < nq; i++) {
// ASSERT_EQ(final_result[i].size(), final_result_3[i].size());
// for (k = 0; k < final_result[i].size(); k++) {
// ASSERT_EQ(final_result[i][k].first, final_result_3[i][k].first);
// ASSERT_EQ(final_result[i][k].second, final_result_3[i][k].second);
// }
// }
// ///////////////////////////////////////////////////////////////////////////////////////
// /* method-2 */
// std::vector<std::vector<int64_t>> id_vec_2(index_file_num);
// std::vector<std::vector<float>> dist_vec_2(index_file_num);
// std::vector<uint64_t> k_vec_2(index_file_num);
// for (i = 0; i < index_file_num; i++) {
// CopyResult(id_vec_2[i], dist_vec_2[i], top_k, id_vec[i], dist_vec[i], TOPK, nq);
// k_vec_2[i] = top_k;
// }
//
// std::string str2 = "Method-2 " + std::to_string(max_thread_num) + " " +
// std::to_string(nq) + " " + std::to_string(top_k);
// milvus::TimeRecorder rc2(str2);
//
// for (step = 1; step < index_file_num; step *= 2) {
// for (i = 0; i + step < index_file_num; i += step * 2) {
// ms::XSearchTask::MergeTopkArray(id_vec_2[i], dist_vec_2[i], k_vec_2[i],
// id_vec_2[i + step], dist_vec_2[i + step],
// k_vec_2[i + step], nq, top_k, ascending);
// }
// }
// ms::XSearchTask::MergeTopkToResultSet(id_vec_2[0],
// dist_vec_2[0],
// k_vec_2[0],
// nq,
// top_k,
// ascending,
// final_result_2);
// ASSERT_EQ(final_result_2.size(), nq);
//
// rc2.RecordSection("reduce done");
//
// for (i = 0; i < nq; i++) {
// ASSERT_EQ(final_result[i].size(), final_result_2[i].size());
// for (k = 0; k < final_result[i].size(); k++) {
// if (final_result[i][k].first != final_result_2[i][k].first) {
// std::cout << i << " " << k << std::endl;
// }
// ASSERT_EQ(final_result[i][k].first, final_result_2[i][k].first);
// ASSERT_EQ(final_result[i][k].second, final_result_2[i][k].second);
// }
// }
//
// ///////////////////////////////////////////////////////////////////////////////////////
// /* method-3 parallel */
// std::vector<std::vector<int64_t>> id_vec_3(index_file_num);
// std::vector<std::vector<float>> dist_vec_3(index_file_num);
// std::vector<uint64_t> k_vec_3(index_file_num);
// for (i = 0; i < index_file_num; i++) {
// CopyResult(id_vec_3[i], dist_vec_3[i], top_k, id_vec[i], dist_vec[i], TOPK, nq);
// k_vec_3[i] = top_k;
// }
//
// std::string str3 = "Method-3 " + std::to_string(max_thread_num) + " " +
// std::to_string(nq) + " " + std::to_string(top_k);
// milvus::TimeRecorder rc3(str3);
//
// for (step = 1; step < index_file_num; step *= 2) {
// for (i = 0; i + step < index_file_num; i += step * 2) {
// threads_list.push_back(
// threadPool.enqueue(ms::XSearchTask::MergeTopkArray,
// std::ref(id_vec_3[i]),
// std::ref(dist_vec_3[i]),
// std::ref(k_vec_3[i]),
// std::ref(id_vec_3[i + step]),
// std::ref(dist_vec_3[i + step]),
// std::ref(k_vec_3[i + step]),
// nq,
// top_k,
// ascending));
// }
//
// while (threads_list.size() > 0) {
// int nready = 0;
// for (auto it = threads_list.begin(); it != threads_list.end(); it = it) {
// auto &p = *it;
// std::chrono::milliseconds span(0);
// if (p.wait_for(span) == std::future_status::ready) {
// threads_list.erase(it++);
// ++nready;
// } else {
// ++it;
// }
// }
//
// if (nready == 0) {
// std::this_thread::yield();
// }
// }
// }
// ms::XSearchTask::MergeTopkToResultSet(id_vec_3[0],
// dist_vec_3[0],
// k_vec_3[0],
// nq,
// top_k,
// ascending,
// final_result_3);
// ASSERT_EQ(final_result_3.size(), nq);
//
// rc3.RecordSection("reduce done");
//
// for (i = 0; i < nq; i++) {
// ASSERT_EQ(final_result[i].size(), final_result_3[i].size());
// for (k = 0; k < final_result[i].size(); k++) {
// ASSERT_EQ(final_result[i][k].first, final_result_3[i][k].first);
// ASSERT_EQ(final_result[i][k].second, final_result_3[i][k].second);
// }
// }
}
}
}

109
core/unittest/db/utils.cpp
View File

@ -15,18 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include <boost/filesystem.hpp>
#include <iostream>
#include <thread>
#include <memory>
#include <string>
#include <boost/filesystem.hpp>
#include <thread>
#include "db/utils.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "db/DBFactory.h"
#include "db/Options.h"
#include "db/utils.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#endif
@ -36,70 +35,71 @@ INITIALIZE_EASYLOGGINGPP
namespace {
static const char
* CONFIG_STR = "# All the following configurations are default values.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus used to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_mem_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" cache_insert_data: false # whether load inserted data into cache\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20\n"
"\n"
"resource_config:\n"
static const char* CONFIG_STR =
"# All the following configurations are default values.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus used to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_mem_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" cache_insert_data: false # whether load inserted data into cache\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20\n"
"\n"
"resource_config:\n"
#ifdef MILVUS_CPU_VERSION
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
#else
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
#endif
void
WriteToFile(const std::string& file_path, const char* content) {
std::fstream fs(file_path.c_str(), std::ios_base::out);
//write data to file
// write data to file
fs << content;
fs.close();
}
class DBTestEnvironment : public ::testing::Environment {
public:
explicit DBTestEnvironment(const std::string& uri)
: uri_(uri) {
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {
}
std::string getURI() const {
std::string
getURI() const {
return uri_;
}
void SetUp() override {
void
SetUp() override {
getURI();
}
@ -109,17 +109,14 @@ class DBTestEnvironment : public ::testing::Environment {
DBTestEnvironment* test_env = nullptr;
} // namespace
} // namespace
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
BaseTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.set(el::Level::Debug,
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
defaultConf.set(el::Level::Debug, el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::Loggers::reconfigureLogger("default", defaultConf);
}
@ -260,7 +257,7 @@ MySqlMetaTest::GetOptions() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int
main(int argc, char **argv) {
main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;

68
core/unittest/db/utils.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gtest/gtest.h>
@ -23,35 +22,42 @@
#include <memory>
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/meta/MySQLMetaImpl.h"
#include "scheduler/SchedInst.h"
#include "db/meta/SqliteMetaImpl.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/SchedInst.h"
#define TIMING
#ifdef TIMING
#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now();
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) \
LOG(DEBUG) << "RUNTIME of " << name << ": " \
<< std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - \
start) \
.count() \
<< " ms ";
#else
#define INIT_TIMER
#define START_TIMER
#define STOP_TIMER(name)
#endif
static const char *CONFIG_PATH = "/tmp/milvus_test";
static const char *CONFIG_FILE = "/server_config.yaml";
static const char* CONFIG_PATH = "/tmp/milvus_test";
static const char* CONFIG_FILE = "/server_config.yaml";
class BaseTest : public ::testing::Test {
protected:
void InitLog();
void
InitLog();
void SetUp() override;
void TearDown() override;
virtual milvus::engine::DBOptions GetOptions();
void
SetUp() override;
void
TearDown() override;
virtual milvus::engine::DBOptions
GetOptions();
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -59,33 +65,38 @@ class DBTest : public BaseTest {
protected:
milvus::engine::DBPtr db_;
void SetUp() override;
void TearDown() override;
void
SetUp() override;
void
TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DBTest2 : public DBTest {
protected:
milvus::engine::DBOptions GetOptions() override;
milvus::engine::DBOptions
GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class EngineTest : public DBTest {
};
class EngineTest : public DBTest {};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MetaTest : public BaseTest {
protected:
std::shared_ptr<milvus::engine::meta::SqliteMetaImpl> impl_;
void SetUp() override;
void TearDown() override;
void
SetUp() override;
void
TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MySqlDBTest : public DBTest {
protected:
milvus::engine::DBOptions GetOptions() override;
milvus::engine::DBOptions
GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -93,15 +104,16 @@ class MySqlMetaTest : public BaseTest {
protected:
std::shared_ptr<milvus::engine::meta::MySQLMetaImpl> impl_;
void SetUp() override;
void TearDown() override;
milvus::engine::DBOptions GetOptions() override;
void
SetUp() override;
void
TearDown() override;
milvus::engine::DBOptions
GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest : public MetaTest {
};
class MemManagerTest : public MetaTest {};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest2 : public DBTest {
};
class MemManagerTest2 : public DBTest {};

4
core/unittest/main.cpp
View File

@ -15,15 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "external/easyloggingpp/easylogging++.h"
INITIALIZE_EASYLOGGINGPP
int
main(int argc, char **argv) {
main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1
core/unittest/metrics/test_metricbase.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/Metrics.h"
#include <gtest/gtest.h>

35
core/unittest/metrics/test_metrics.cpp
View File

@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <chrono>
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <gtest/gtest.h>
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "server/Config.h"
TEST_F(MetricTest, METRIC_TEST) {
milvus::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
@ -36,15 +36,15 @@ TEST_F(MetricTest, METRIC_TEST) {
milvus::server::Metrics::GetInstance();
milvus::server::SystemInfo::GetInstance().Init();
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
milvus::server::Metrics::GetInstance().Init();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char *group_name = "test_group";
static const char* group_name = "test_group";
static const int group_dim = 256;
milvus::engine::meta::TableSchema group_info;
@ -61,14 +61,14 @@ TEST_F(MetricTest, METRIC_TEST) {
int d = 256;
int nb = 50;
float *xb = new float[d * nb];
float* xb = new float[d * nb];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) xb[d * i + j] = drand48();
xb[d * i] += i / 2000.;
}
int qb = 5;
float *qxb = new float[d * qb];
float* qxb = new float[d * qb];
for (int i = 0; i < qb; i++) {
for (int j = 0; j < d; j++) qxb[d * i + j] = drand48();
qxb[d * i] += i / 2000.;
@ -90,17 +90,16 @@ TEST_F(MetricTest, METRIC_TEST) {
prev_count = count;
START_TIMER;
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count * group_dim * sizeof(float)) / (1024 * 1024) << " M";
for (auto k = 0; k < qb; ++k) {
// ASSERT_EQ(results[k][0].first, target_ids[k]);
// ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
}
ASSERT_TRUE(count >= prev_count);
std::this_thread::sleep_for(std::chrono::seconds(1));
@ -153,5 +152,3 @@ TEST_F(MetricTest, COLLECTOR_METRICS_TEST) {
milvus::server::MetricCollector metric_collector();
}

1
core/unittest/metrics/test_prometheus.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "server/Config.h"

38
core/unittest/metrics/utils.cpp
View File

@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include <iostream>
#include <thread>
#include <string>
#include <boost/filesystem.hpp>
#include <iostream>
#include <string>
#include <thread>
#include "metrics/utils.h"
#include "db/DBFactory.h"
#include "metrics/utils.h"
INITIALIZE_EASYLOGGINGPP
@ -30,13 +29,16 @@ namespace {
class DBTestEnvironment : public ::testing::Environment {
public:
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {}
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {
}
std::string getURI() const {
std::string
getURI() const {
return uri_;
}
void SetUp() override {
void
SetUp() override {
getURI();
}
@ -46,36 +48,40 @@ class DBTestEnvironment : public ::testing::Environment {
DBTestEnvironment* test_env = nullptr;
} // namespace
} // namespace
void MetricTest::InitLog() {
void
MetricTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.set(el::Level::Debug,
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
defaultConf.set(el::Level::Debug, el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::Loggers::reconfigureLogger("default", defaultConf);
}
milvus::engine::DBOptions MetricTest::GetOptions() {
milvus::engine::DBOptions
MetricTest::GetOptions() {
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
void MetricTest::SetUp() {
void
MetricTest::SetUp() {
boost::filesystem::remove_all("/tmp/milvus_test");
InitLog();
auto options = GetOptions();
db_ = milvus::engine::DBFactory::Build(options);
}
void MetricTest::TearDown() {
void
MetricTest::TearDown() {
db_->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
}
int main(int argc, char **argv) {
int
main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;

32
core/unittest/metrics/utils.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gtest/gtest.h>
@ -23,17 +22,20 @@
//#include <src/db/MySQLMetaImpl.h>
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/meta/SqliteMetaImpl.h"
#define TIMING
#ifdef TIMING
#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now();
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) \
LOG(DEBUG) << "RUNTIME of " << name << ": " \
<< std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - \
start) \
.count() \
<< " ms ";
#else
#define INIT_TIMER
#define START_TIMER
@ -41,10 +43,10 @@
#endif
void
ASSERT_STATS(milvus::Status &stat);
ASSERT_STATS(milvus::Status& stat);
//class TestEnv : public ::testing::Environment {
//public:
// class TestEnv : public ::testing::Environment {
// public:
//
// static std::string getURI() {
// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) {
@ -68,8 +70,12 @@ class MetricTest : public ::testing::Test {
protected:
milvus::engine::DBPtr db_;
void InitLog();
void SetUp() override;
void TearDown() override;
virtual milvus::engine::DBOptions GetOptions();
void
InitLog();
void
SetUp() override;
void
TearDown() override;
virtual milvus::engine::DBOptions
GetOptions();
};

12
core/unittest/scheduler/task_test.cpp
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/task/SearchTask.h"
#include "scheduler/task/BuildIndexTask.h"
#include <gtest/gtest.h>
#include "scheduler/task/BuildIndexTask.h"
#include "scheduler/task/SearchTask.h"
namespace milvus {
namespace scheduler {
@ -34,7 +32,5 @@ TEST(TaskTest, INVALID_INDEX) {
build_task->Execute();
}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

13
core/unittest/scheduler/test_algorithm.cpp
View File

@ -15,15 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/resource/Resource.h"
#include "scheduler/Algorithm.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/ResourceMgr.h"
#include "scheduler/resource/CpuResource.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/Algorithm.h"
#include "scheduler/resource/Resource.h"
namespace milvus {
namespace scheduler {
@ -103,6 +101,5 @@ TEST_F(AlgorithmTest, SHORTESTPATH_TEST) {
}
}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

13
core/unittest/scheduler/test_event.cpp
View File

@ -15,14 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/resource/Resource.h"
#include "scheduler/event/Event.h"
#include "scheduler/event/StartUpEvent.h"
#include "scheduler/resource/Resource.h"
namespace milvus {
namespace scheduler {
@ -51,7 +47,6 @@ TEST(EventTest, FINISH_TASK_EVENT) {
std::cout << *EventPtr(event);
}
TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
ResourcePtr res(nullptr);
auto event = std::make_shared<TaskTableUpdatedEvent>(res);
@ -60,7 +55,5 @@ TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
std::cout << *EventPtr(event);
}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

9
core/unittest/scheduler/test_normal.cpp
View File

@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/ResourceFactory.h"
#include "scheduler/ResourceMgr.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Scheduler.h"
#include "scheduler/task/TestTask.h"
#include "scheduler/tasklabel/DefaultLabel.h"
#include "scheduler/SchedInst.h"
#include "utils/Log.h"
#include <gtest/gtest.h>
namespace {
namespace ms = milvus::scheduler;
} // namespace
} // namespace
TEST(NormalTest, INST_TEST) {
// ResourceMgr only compose resources, provide unified event
@ -62,7 +61,7 @@ TEST(NormalTest, INST_TEST) {
}
}
for (auto &task : tasks) {
for (auto& task : tasks) {
task->Wait();
ASSERT_EQ(task->load_count_, 1);
ASSERT_EQ(task->exec_count_, 1);

23
core/unittest/scheduler/test_resource.cpp
View File

@ -15,18 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/resource/Resource.h"
#include "scheduler/resource/DiskResource.h"
#include <gtest/gtest.h>
#include "scheduler/ResourceFactory.h"
#include "scheduler/resource/CpuResource.h"
#include "scheduler/resource/DiskResource.h"
#include "scheduler/resource/GpuResource.h"
#include "scheduler/resource/Resource.h"
#include "scheduler/resource/TestResource.h"
#include "scheduler/task/Task.h"
#include "scheduler/task/TestTask.h"
#include "scheduler/tasklabel/DefaultLabel.h"
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
namespace milvus {
namespace scheduler {
@ -158,17 +156,13 @@ class ResourceAdvanceTest : public testing::Test {
void
WaitLoader(uint64_t count) {
std::unique_lock<std::mutex> lock(load_mutex_);
cv_.wait(lock, [&] {
return load_count_ == count;
});
cv_.wait(lock, [&] { return load_count_ == count; });
}
void
WaitExecutor(uint64_t count) {
std::unique_lock<std::mutex> lock(exec_mutex_);
cv_.wait(lock, [&] {
return exec_count_ == count;
});
cv_.wait(lock, [&] { return exec_count_ == count; });
}
ResourcePtr disk_resource_;
@ -287,6 +281,5 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
}
}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

5
core/unittest/scheduler/test_resource_factory.cpp
View File

@ -15,15 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
#include "scheduler/ResourceFactory.h"
namespace {
namespace ms = milvus::scheduler;
} // namespace
} // namespace
TEST(ResourceFactoryTest, CREATE) {
auto disk = ms::ResourceFactory::Create("ssd", "DISK", 0);

28
core/unittest/scheduler/test_resource_mgr.cpp
View File

@ -15,16 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/ResourceMgr.h"
#include "scheduler/resource/CpuResource.h"
#include "scheduler/resource/GpuResource.h"
#include "scheduler/resource/DiskResource.h"
#include "scheduler/resource/GpuResource.h"
#include "scheduler/resource/TestResource.h"
#include "scheduler/task/TestTask.h"
#include "scheduler/tasklabel/DefaultLabel.h"
#include "scheduler/ResourceMgr.h"
#include <gtest/gtest.h>
namespace milvus {
namespace scheduler {
@ -101,10 +99,13 @@ TEST_F(ResourceMgrBaseTest, GET_ALL_RESOURCES) {
bool disk = false, cpu = false, gpu = false;
auto resources = mgr1_->GetAllResources();
ASSERT_EQ(resources.size(), 3);
for (auto &res : resources) {
if (res->type() == ResourceType::DISK) disk = true;
if (res->type() == ResourceType::CPU) cpu = true;
if (res->type() == ResourceType::GPU) gpu = true;
for (auto& res : resources) {
if (res->type() == ResourceType::DISK)
disk = true;
if (res->type() == ResourceType::CPU)
cpu = true;
if (res->type() == ResourceType::GPU)
gpu = true;
}
ASSERT_TRUE(disk);
@ -183,9 +184,7 @@ class ResourceMgrAdvanceTest : public testing::Test {
TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
bool flag = false;
auto callback = [&](EventPtr event) {
flag = true;
};
auto callback = [&](EventPtr event) { flag = true; };
mgr1_->RegisterSubscriber(callback);
TableFileSchemaPtr dummy = nullptr;
auto label = std::make_shared<DefaultLabel>();
@ -194,6 +193,5 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
ASSERT_TRUE(flag);
}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

71
core/unittest/scheduler/test_scheduler.cpp
View File

@ -17,14 +17,14 @@
#include <gtest/gtest.h>
#include "scheduler/Scheduler.h"
#include "scheduler/tasklabel/DefaultLabel.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "scheduler/task/TestTask.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/resource/Resource.h"
#include "cache/DataObj.h"
#include "cache/GpuCacheMgr.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/Scheduler.h"
#include "scheduler/resource/Resource.h"
#include "scheduler/task/TestTask.h"
#include "scheduler/tasklabel/DefaultLabel.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
@ -33,59 +33,61 @@ namespace scheduler {
class MockVecIndex : public engine::VecIndex {
public:
virtual Status BuildAll(const int64_t& nb,
const float* xb,
const int64_t* ids,
const engine::Config& cfg,
const int64_t& nt = 0,
const float* xt = nullptr) {
virtual Status
BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const engine::Config& cfg, const int64_t& nt = 0,
const float* xt = nullptr) {
}
engine::VecIndexPtr Clone() override {
engine::VecIndexPtr
Clone() override {
return milvus::engine::VecIndexPtr();
}
int64_t GetDeviceId() override {
int64_t
GetDeviceId() override {
return 0;
}
engine::IndexType GetType() override {
engine::IndexType
GetType() override {
return engine::IndexType::INVALID;
}
virtual Status Add(const int64_t& nb,
const float* xb,
const int64_t* ids,
const engine::Config& cfg = engine::Config()) {
virtual Status
Add(const int64_t& nb, const float* xb, const int64_t* ids, const engine::Config& cfg = engine::Config()) {
}
virtual Status Search(const int64_t& nq,
const float* xq,
float* dist,
int64_t* ids,
const engine::Config& cfg = engine::Config()) {
virtual Status
Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids,
const engine::Config& cfg = engine::Config()) {
}
engine::VecIndexPtr CopyToGpu(const int64_t& device_id, const engine::Config& cfg) override {
engine::VecIndexPtr
CopyToGpu(const int64_t& device_id, const engine::Config& cfg) override {
}
engine::VecIndexPtr CopyToCpu(const engine::Config& cfg) override {
engine::VecIndexPtr
CopyToCpu(const engine::Config& cfg) override {
}
virtual int64_t Dimension() {
virtual int64_t
Dimension() {
return dimension_;
}
virtual int64_t Count() {
virtual int64_t
Count() {
return ntotal_;
}
virtual knowhere::BinarySet Serialize() {
virtual knowhere::BinarySet
Serialize() {
knowhere::BinarySet binset;
return binset;
}
virtual Status Load(const knowhere::BinarySet& index_binary) {
virtual Status
Load(const knowhere::BinarySet& index_binary) {
}
public:
@ -185,7 +187,7 @@ TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
}
sleep(3);
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
}
class SchedulerTest2 : public testing::Test {
@ -240,7 +242,7 @@ class SchedulerTest2 : public testing::Test {
std::shared_ptr<Scheduler> scheduler_;
};
//TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
// TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
// const uint64_t NUM = 2;
// std::vector<std::shared_ptr<TestTask>> tasks;
// TableFileSchemaPtr dummy = std::make_shared<TableFileSchema>();
@ -257,6 +259,5 @@ class SchedulerTest2 : public testing::Test {
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
//}
} // namespace scheduler
} // namespace milvus
} // namespace scheduler
} // namespace milvus

68
core/unittest/server/test_cache.cpp
View File

@ -38,70 +38,70 @@ class LessItemCacheMgr : public milvus::cache::CacheMgr<milvus::cache::DataObjPt
class MockVecIndex : public milvus::engine::VecIndex {
public:
MockVecIndex(int64_t dim, int64_t total)
: dimension_(dim),
ntotal_(total) {
MockVecIndex(int64_t dim, int64_t total) : dimension_(dim), ntotal_(total) {
}
virtual milvus::Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const milvus::engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
virtual milvus::Status
BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const milvus::engine::Config& cfg,
const int64_t& nt = 0, const float* xt = nullptr) {
return milvus::Status();
}
milvus::engine::VecIndexPtr Clone() override {
milvus::engine::VecIndexPtr
Clone() override {
return milvus::engine::VecIndexPtr();
}
int64_t GetDeviceId() override {
int64_t
GetDeviceId() override {
return 0;
}
milvus::engine::IndexType GetType() override {
milvus::engine::IndexType
GetType() override {
return milvus::engine::IndexType::INVALID;
}
virtual milvus::Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const milvus::engine::Config &cfg = milvus::engine::Config()) {
virtual milvus::Status
Add(const int64_t& nb, const float* xb, const int64_t* ids,
const milvus::engine::Config& cfg = milvus::engine::Config()) {
return milvus::Status();
}
virtual milvus::Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const milvus::engine::Config &cfg = milvus::engine::Config()) {
virtual milvus::Status
Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids,
const milvus::engine::Config& cfg = milvus::engine::Config()) {
return milvus::Status();
}
milvus::engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const milvus::engine::Config &cfg) override {
milvus::engine::VecIndexPtr
CopyToGpu(const int64_t& device_id, const milvus::engine::Config& cfg) override {
return nullptr;
}
milvus::engine::VecIndexPtr CopyToCpu(const milvus::engine::Config &cfg) override {
milvus::engine::VecIndexPtr
CopyToCpu(const milvus::engine::Config& cfg) override {
return nullptr;
}
virtual int64_t Dimension() {
virtual int64_t
Dimension() {
return dimension_;
}
virtual int64_t Count() {
virtual int64_t
Count() {
return ntotal_;
}
virtual knowhere::BinarySet Serialize() {
virtual knowhere::BinarySet
Serialize() {
knowhere::BinarySet binset;
return binset;
}
virtual milvus::Status Load(const knowhere::BinarySet &index_binary) {
virtual milvus::Status
Load(const knowhere::BinarySet& index_binary) {
return milvus::Status();
}
@ -110,7 +110,7 @@ class MockVecIndex : public milvus::engine::VecIndex {
int64_t ntotal_ = 0;
};
} // namespace
} // namespace
TEST(CacheTest, DUMMY_TEST) {
milvus::engine::Config cfg;
@ -141,7 +141,7 @@ TEST(CacheTest, CPU_CACHE_TEST) {
uint64_t item_count = 20;
for (uint64_t i = 0; i < item_count; i++) {
//each vector is 1k byte, total size less than 1G
// each vector is 1k byte, total size less than 1G
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
@ -165,7 +165,7 @@ TEST(CacheTest, CPU_CACHE_TEST) {
g_num = 5;
cpu_mgr->SetCapacity(g_num * gbyte);
//each vector is 1k byte, total size less than 6G
// each vector is 1k byte, total size less than 6G
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_6g", data_obj);
@ -179,7 +179,7 @@ TEST(CacheTest, GPU_CACHE_TEST) {
auto gpu_mgr = milvus::cache::GpuCacheMgr::GetInstance(0);
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
// each vector is 1k byte
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
@ -192,7 +192,7 @@ TEST(CacheTest, GPU_CACHE_TEST) {
for (auto i = 0; i < 3; i++) {
// TODO(myh): use gpu index to mock
//each vector is 1k byte, total size less than 2G
// each vector is 1k byte, total size less than 2G
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
std::cout << data_obj->Size() << std::endl;
@ -223,7 +223,7 @@ TEST(CacheTest, INVALID_TEST) {
{
LessItemCacheMgr mgr;
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
// each vector is 1k byte
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
mgr.InsertItem("index_" + std::to_string(i), data_obj);

8
core/unittest/server/test_config.cpp
View File

@ -15,14 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <gtest/gtest-death-test.h>
#include <gtest/gtest.h>
#include "config/YamlConfigMgr.h"
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
#include "server/Config.h"
#include "server/utils.h"
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
namespace {
@ -30,7 +30,7 @@ static constexpr uint64_t KB = 1024;
static constexpr uint64_t MB = KB * 1024;
static constexpr uint64_t GB = MB * 1024;
} // namespace
} // namespace
TEST_F(ConfigTest, CONFIG_TEST) {
milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance();

100
core/unittest/server/test_rpc.cpp
View File

@ -16,8 +16,8 @@
// under the License.
#include <gtest/gtest.h>
#include <thread>
#include <boost/filesystem.hpp>
#include <thread>
#include "server/Server.h"
#include "server/grpc_impl/GrpcRequestHandler.h"
@ -28,10 +28,10 @@
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
#include "server/DBWrapper.h"
#include "server/Config.h"
#include "scheduler/SchedInst.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "utils/CommonUtil.h"
namespace {
@ -71,18 +71,18 @@ class RpcHandlerTest : public testing::Test {
milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
//
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "read_only");
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
//
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "read_only");
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
milvus::server::Config::GetInstance().SetResourceConfigMode("single");
milvus::server::DBWrapper::GetInstance().StartService();
//initialize handler, create table
// initialize handler, create table
handler = std::make_shared<milvus::server::grpc::GrpcRequestHandler>();
::grpc::ServerContext context;
::milvus::grpc::TableSchema request;
@ -108,8 +108,7 @@ class RpcHandlerTest : public testing::Test {
};
void
BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >>& vector_record_array) {
BuildVectors(int64_t from, int64_t to, std::vector<std::vector<float>>& vector_record_array) {
if (to <= from) {
return;
}
@ -135,13 +134,13 @@ CurrentTmDate(int64_t offset_day = 0) {
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
std::string str =
std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday);
return str;
}
} // namespace
} // namespace
TEST_F(RpcHandlerTest, HAS_TABLE_TEST) {
::grpc::ServerContext context;
@ -173,7 +172,7 @@ TEST_F(RpcHandlerTest, INDEX_TEST) {
grpc_status = handler->CreateIndex(&context, &request, &response);
ASSERT_EQ(grpc_status.error_code(), ::grpc::Status::OK.error_code());
int error_code = response.error_code();
// ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
// ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
::milvus::grpc::TableName table_name;
::milvus::grpc::IndexParam index_param;
@ -214,25 +213,25 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
::grpc::ServerContext context;
::milvus::grpc::SearchParam request;
::milvus::grpc::TopKQueryResultList response;
//test null input
// test null input
handler->Search(&context, nullptr, &response);
//test invalid table name
// test invalid table name
handler->Search(&context, &request, &response);
//test table not exist
// test table not exist
request.set_table_name("test3");
handler->Search(&context, &request, &response);
//test invalid topk
// test invalid topk
request.set_table_name(TABLE_NAME);
handler->Search(&context, &request, &response);
//test invalid nprobe
// test invalid nprobe
request.set_topk(10);
handler->Search(&context, &request, &response);
//test empty query record array
// test empty query record array
request.set_nprobe(32);
handler->Search(&context, &request, &response);
@ -245,7 +244,7 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
grpc_record->add_vector_data(record[i]);
}
}
//insert vectors
// insert vectors
insert_param.set_table_name(TABLE_NAME);
::milvus::grpc::VectorIds vector_ids;
handler->Insert(&context, &insert_param, &vector_ids);
@ -260,7 +259,7 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
}
handler->Search(&context, &request, &response);
//test search with range
// test search with range
::milvus::grpc::Range* range = request.mutable_query_range_array()->Add();
range->set_start_value(CurrentTmDate(-2));
range->set_end_value(CurrentTmDate(-3));
@ -284,26 +283,26 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
::milvus::grpc::Status response;
std::string tablename = "tbl";
//create table test
//test null input
// create table test
// test null input
handler->CreateTable(&context, nullptr, &response);
//test invalid table name
// test invalid table name
handler->CreateTable(&context, &tableschema, &response);
//test invalid table dimension
// test invalid table dimension
tableschema.set_table_name(tablename);
handler->CreateTable(&context, &tableschema, &response);
//test invalid index file size
// test invalid index file size
tableschema.set_dimension(TABLE_DIM);
// handler->CreateTable(&context, &tableschema, &response);
//test invalid index metric type
// handler->CreateTable(&context, &tableschema, &response);
// test invalid index metric type
tableschema.set_index_file_size(INDEX_FILE_SIZE);
handler->CreateTable(&context, &tableschema, &response);
//test table already exist
// test table already exist
tableschema.set_metric_type(1);
handler->CreateTable(&context, &tableschema, &response);
//describe table test
//test invalid table name
// describe table test
// test invalid table name
::milvus::grpc::TableName table_name;
::milvus::grpc::TableSchema table_schema;
handler->DescribeTable(&context, &table_name, &table_schema);
@ -316,11 +315,11 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
::milvus::grpc::VectorIds vector_ids;
//Insert vectors
//test invalid table name
// Insert vectors
// test invalid table name
handler->Insert(&context, &request, &vector_ids);
request.set_table_name(tablename);
//test empty row record
// test empty row record
handler->Insert(&context, &request, &vector_ids);
for (auto& record : record_array) {
@ -329,12 +328,12 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
grpc_record->add_vector_data(record[i]);
}
}
//test vector_id size not equal to row record size
// test vector_id size not equal to row record size
vector_ids.clear_vector_id_array();
vector_ids.add_vector_id_array(1);
handler->Insert(&context, &request, &vector_ids);
//normally test
// normally test
vector_ids.clear_vector_id_array();
handler->Insert(&context, &request, &vector_ids);
@ -348,33 +347,31 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
}
handler->Insert(&context, &request, &vector_ids);
//show tables
// show tables
::milvus::grpc::Command cmd;
::milvus::grpc::TableNameList table_name_list;
status = handler->ShowTables(&context, &cmd, &table_name_list);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
//Count Table
// Count Table
::milvus::grpc::TableRowCount count;
table_name.Clear();
status = handler->CountTable(&context, &table_name, &count);
table_name.set_table_name(tablename);
status = handler->CountTable(&context, &table_name, &count);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
// ASSERT_EQ(count.table_row_count(), vector_ids.vector_id_array_size());
// ASSERT_EQ(count.table_row_count(), vector_ids.vector_id_array_size());
//Preload Table
// Preload Table
table_name.Clear();
status = handler->PreloadTable(&context, &table_name, &response);
table_name.set_table_name(TABLE_NAME);
status = handler->PreloadTable(&context, &table_name, &response);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
//Drop table
// Drop table
table_name.set_table_name("");
//test invalid table name
// test invalid table name
::grpc::Status grpc_status = handler->DropTable(&context, &table_name, &response);
table_name.set_table_name(tablename);
grpc_status = handler->DropTable(&context, &table_name, &response);
@ -410,7 +407,7 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
::grpc::Status grpc_status = handler->DeleteByRange(&context, &request, &status);
int error_code = status.error_code();
// ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
// ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
request.mutable_range()->set_start_value("test6");
grpc_status = handler->DeleteByRange(&context, &request, &status);
@ -451,7 +448,7 @@ class RpcSchedulerTest : public testing::Test {
std::shared_ptr<DummyTask> task_ptr;
};
} // namespace
} // namespace
TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
auto status = task_ptr->Execute();
@ -469,4 +466,3 @@ TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
milvus::server::grpc::GrpcRequestScheduler::GetInstance().Stop();
}

48
core/unittest/server/test_util.cpp
View File

@ -15,21 +15,21 @@
// specific language governing permissions and limitations
// under the License.
#include "utils/SignalUtil.h"
#include "db/engine/ExecutionEngine.h"
#include "utils/BlockingQueue.h"
#include "utils/CommonUtil.h"
#include "utils/Error.h"
#include "utils/LogUtil.h"
#include "utils/SignalUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/TimeRecorder.h"
#include "utils/BlockingQueue.h"
#include "utils/LogUtil.h"
#include "utils/ValidationUtil.h"
#include "db/engine/ExecutionEngine.h"
#include <thread>
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/filesystem.hpp>
#include <gtest/gtest.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <boost/filesystem.hpp>
#include <thread>
namespace {
@ -40,7 +40,7 @@ CopyStatus(milvus::Status& st1, milvus::Status& st2) {
st1 = st2;
}
} // namespace
} // namespace
TEST(UtilTest, EXCEPTION_TEST) {
std::string err_msg = "failed";
@ -69,7 +69,7 @@ TEST(UtilTest, COMMON_TEST) {
std::string path3 = path2 + "abcdef";
milvus::Status status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
//test again
// test again
status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
@ -77,7 +77,7 @@ TEST(UtilTest, COMMON_TEST) {
status = milvus::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
//test again
// test again
status = milvus::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
@ -181,7 +181,7 @@ TEST(UtilTest, LOG_TEST) {
TEST(UtilTest, TIMERECORDER_TEST) {
for (int64_t log_level = 0; log_level <= 6; log_level++) {
if (log_level == 5) {
continue; //skip fatal
continue; // skip fatal
}
milvus::TimeRecorder rc("time", log_level);
rc.RecordSection("end");
@ -282,9 +282,9 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
#endif
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
}
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(
(int)milvus::engine::EngineType::MAX_VALUE + 1).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(
milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::MAX_VALUE + 1).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(0).code(), milvus::SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(100).code(), milvus::SERVER_SUCCESS);
@ -360,21 +360,11 @@ TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) {
TEST(UtilTest, ROLLOUTHANDLER_TEST) {
std::string dir1 = "/tmp/milvus_test";
std::string dir2 = "/tmp/milvus_test/log_test";
std::string filename[6] = {
"log_global.log",
"log_debug.log",
"log_warning.log",
"log_trace.log",
"log_error.log",
"log_fatal.log"};
std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log",
"log_trace.log", "log_error.log", "log_fatal.log"};
el::Level list[6] = {
el::Level::Global,
el::Level::Debug,
el::Level::Warning,
el::Level::Trace,
el::Level::Error,
el::Level::Fatal};
el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning,
el::Level::Trace, el::Level::Error, el::Level::Fatal};
mkdir(dir1.c_str(), S_IRWXU);
mkdir(dir2.c_str(), S_IRWXU);

91
core/unittest/server/utils.cpp
View File

@ -20,54 +20,54 @@
#include <fstream>
#include <iostream>
#include <thread>
#include <string>
#include <thread>
namespace {
static const char
* VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
" preload_table: \n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus uses to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_cache_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_cache_threshold: 0.85 \n"
" gpu_cache_capacity: 4 # GB, GPU memory used for cache\n"
" gpu_cache_threshold: 0.85 \n"
" cache_insert_data: false # whether to load inserted data into cache\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20 \n"
"\n"
"resource_config:\n"
static const char* VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
" preload_table: \n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus uses to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_cache_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_cache_threshold: 0.85 \n"
" gpu_cache_capacity: 4 # GB, GPU memory used for cache\n"
" gpu_cache_threshold: 0.85 \n"
" cache_insert_data: false # whether to load inserted data into cache\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20 \n"
"\n"
"resource_config:\n"
#ifdef MILVUS_CPU_VERSION
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
#else
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
#endif
static const char* INVALID_CONFIG_STR = "*INVALID*";
@ -76,20 +76,19 @@ void
WriteToFile(const std::string& file_path, const char* content) {
std::fstream fs(file_path.c_str(), std::ios_base::out);
//write data to file
// write data to file
fs << content;
fs.close();
}
} // namespace
} // namespace
void
ConfigTest::SetUp() {
std::string config_path(CONFIG_PATH);
milvus::server::CommonUtil::CreateDirectory(config_path);
WriteToFile(config_path + VALID_CONFIG_FILE, VALID_CONFIG_STR);
WriteToFile(config_path+ INVALID_CONFIG_FILE, INVALID_CONFIG_STR);
WriteToFile(config_path + INVALID_CONFIG_FILE, INVALID_CONFIG_STR);
}
void

13
core/unittest/server/utils.h
View File

@ -15,18 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gtest/gtest.h>
#include <chrono>
static const char *CONFIG_PATH = "/tmp/milvus_test/";
static const char *VALID_CONFIG_FILE = "valid_config.yaml";
static const char *INVALID_CONFIG_FILE = "invalid_config.conf";
static const char* CONFIG_PATH = "/tmp/milvus_test/";
static const char* VALID_CONFIG_FILE = "valid_config.yaml";
static const char* INVALID_CONFIG_FILE = "invalid_config.conf";
class ConfigTest : public ::testing::Test {
protected:
void SetUp() override;
void TearDown() override;
void
SetUp() override;
void
TearDown() override;
};

15
core/unittest/wrapper/test_hybrid_index.cpp
View File

@ -15,22 +15,22 @@
// specific language governing permissions and limitations
// under the License.
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include <gtest/gtest.h>
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
using ::testing::Combine;
using ::testing::TestWithParam;
using ::testing::Values;
using ::testing::Combine;
class KnowhereHybrid
: public DataGenBase, public ::testing::Test {
class KnowhereHybrid : public DataGenBase, public ::testing::Test {
protected:
void SetUp() override {
void
SetUp() override {
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM);
dim = 128;
@ -40,7 +40,8 @@ class KnowhereHybrid
GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
}
void TearDown() override {
void
TearDown() override {
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}

2
core/unittest/wrapper/test_knowhere.cpp
View File

@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "server/Config.h"
#include "wrapper/KnowhereResource.h"
#include "wrapper/utils.h"
#include "server/Config.h"
#include <gtest/gtest.h>

46
core/unittest/wrapper/test_wrapper.cpp
View File

@ -29,15 +29,16 @@
INITIALIZE_EASYLOGGINGPP
using ::testing::Combine;
using ::testing::TestWithParam;
using ::testing::Values;
using ::testing::Combine;
class KnowhereWrapperTest
: public DataGenBase,
public TestWithParam<::std::tuple<milvus::engine::IndexType, std::string, int, int, int, int>> {
protected:
void SetUp() override {
: public DataGenBase,
public TestWithParam<::std::tuple<milvus::engine::IndexType, std::string, int, int, int, int>> {
protected:
void
SetUp() override {
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM);
#endif
@ -53,45 +54,36 @@ protected:
conf->gpu_id = DEVICEID;
}
void TearDown() override {
void
TearDown() override {
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().Free();
#endif
}
protected:
protected:
milvus::engine::IndexType index_type;
milvus::engine::VecIndexPtr index_ = nullptr;
knowhere::Config conf;
};
INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
Values(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
INSTANTIATE_TEST_CASE_P(
WrapperParam, KnowhereWrapperTest,
Values(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
#ifdef MILVUS_GPU_VERSION
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX,
"Default",
64,
100000,
10,
10),
// std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10),
// std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB,
// 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
#endif
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU,
"Default",
64,
100000,
10,
10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10)
)
);
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10)));
TEST_P(KnowhereWrapperTest, BASE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);

118
core/unittest/wrapper/utils.cpp
View File

@ -15,69 +15,68 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <faiss/IndexFlat.h>
#include <gtest/gtest.h>
#include <string>
#include "wrapper/utils.h"
#include "utils/CommonUtil.h"
#include "wrapper/utils.h"
namespace {
static const char
* CONFIG_STR = "# All the following configurations are default values.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus used to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_mem_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" cache_insert_data: false # whether load inserted data into cache\n"
"\n"
"engine_config:\n"
" blas_threshold: 20\n"
"\n"
"resource_config:\n"
static const char* CONFIG_STR =
"# All the following configurations are default values.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus used to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_mem_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" cache_insert_data: false # whether load inserted data into cache\n"
"\n"
"engine_config:\n"
" blas_threshold: 20\n"
"\n"
"resource_config:\n"
#ifdef MILVUS_CPU_VERSION
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
" search_resources:\n"
" - cpu\n"
" index_build_device: cpu # CPU used for building index";
#else
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
" search_resources:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
#endif
void
WriteToFile(const std::string& file_path, const char* content) {
std::fstream fs(file_path.c_str(), std::ios_base::out);
//write data to file
// write data to file
fs << content;
fs.close();
}
} // namespace
} // namespace
void
KnowhereTest::SetUp() {
@ -93,12 +92,11 @@ KnowhereTest::TearDown() {
}
void
DataGenBase::GenData(const int& dim, const int& nb, const int& nq,
float* xb, float* xq, int64_t* ids,
const int& k, int64_t* gt_ids, float* gt_dis) {
DataGenBase::GenData(const int& dim, const int& nb, const int& nq, float* xb, float* xq, int64_t* ids, const int& k,
int64_t* gt_ids, float* gt_dis) {
for (auto i = 0; i < nb; ++i) {
for (auto j = 0; j < dim; ++j) {
//p_data[i * d + j] = float(base + i);
// p_data[i * d + j] = float(base + i);
xb[i * dim + j] = drand48();
}
xb[dim * i] += i / 1000.;
@ -109,20 +107,14 @@ DataGenBase::GenData(const int& dim, const int& nb, const int& nq,
}
faiss::IndexFlatL2 index(dim);
//index.add_with_ids(nb, xb, ids);
// index.add_with_ids(nb, xb, ids);
index.add(nb, xb);
index.search(nq, xq, k, gt_dis, gt_ids);
}
void
DataGenBase::GenData(const int& dim,
const int& nb,
const int& nq,
std::vector<float>& xb,
std::vector<float>& xq,
std::vector<int64_t>& ids,
const int& k,
std::vector<int64_t>& gt_ids,
DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector<float>& xb, std::vector<float>& xq,
std::vector<int64_t>& ids, const int& k, std::vector<int64_t>& gt_ids,
std::vector<float>& gt_dis) {
xb.resize(nb * dim);
xq.resize(nq * dim);
@ -139,22 +131,20 @@ DataGenBase::AssertResult(const std::vector<int64_t>& ids, const std::vector<flo
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(ids[i * k], gt_ids[i * k]);
//EXPECT_EQ(dis[i * k], gt_dis[i * k]);
// EXPECT_EQ(dis[i * k], gt_dis[i * k]);
}
int match = 0;
for (int i = 0; i < nq; ++i) {
for (int j = 0; j < k; ++j) {
for (int l = 0; l < k; ++l) {
if (ids[i * nq + j] == gt_ids[i * nq + l]) match++;
if (ids[i * nq + j] == gt_ids[i * nq + l])
match++;
}
}
}
auto precision = float(match) / (nq * k);
EXPECT_GT(precision, 0.5);
std::cout << std::endl << "Precision: " << precision
<< ", match: " << match
<< ", total: " << nq * k
<< std::endl;
std::cout << std::endl << "Precision: " << precision << ", match: " << match << ", total: " << nq * k << std::endl;
}

93
core/unittest/wrapper/utils.h
View File

@ -15,19 +15,18 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gtest/gtest.h>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <memory>
#include <vector>
#include <cstdlib>
#include <cstdio>
#include <fstream>
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
class DataGenBase;
@ -41,31 +40,29 @@ constexpr int64_t PINMEM = 1024 * 1024 * 200;
constexpr int64_t TEMPMEM = 1024 * 1024 * 300;
constexpr int64_t RESNUM = 2;
static const char *CONFIG_PATH = "/tmp/milvus_test";
static const char *CONFIG_FILE = "/server_config.yaml";
static const char* CONFIG_PATH = "/tmp/milvus_test";
static const char* CONFIG_FILE = "/server_config.yaml";
class KnowhereTest : public ::testing::Test {
protected:
void SetUp() override;
void TearDown() override;
void
SetUp() override;
void
TearDown() override;
};
class DataGenBase {
public:
virtual void GenData(const int& dim, const int& nb, const int& nq, float* xb, float* xq, int64_t* ids,
const int& k, int64_t* gt_ids, float* gt_dis);
virtual void
GenData(const int& dim, const int& nb, const int& nq, float* xb, float* xq, int64_t* ids, const int& k,
int64_t* gt_ids, float* gt_dis);
virtual void GenData(const int& dim,
const int& nb,
const int& nq,
std::vector<float>& xb,
std::vector<float>& xq,
std::vector<int64_t>& ids,
const int& k,
std::vector<int64_t>& gt_ids,
std::vector<float>& gt_dis);
virtual void
GenData(const int& dim, const int& nb, const int& nq, std::vector<float>& xb, std::vector<float>& xq,
std::vector<int64_t>& ids, const int& k, std::vector<int64_t>& gt_ids, std::vector<float>& gt_dis);
void AssertResult(const std::vector<int64_t>& ids, const std::vector<float>& dis);
void
AssertResult(const std::vector<int64_t>& ids, const std::vector<float>& dis);
int dim = DIM;
int nb = NB;
@ -82,12 +79,14 @@ class DataGenBase {
class ParamGenerator {
public:
static ParamGenerator& GetInstance() {
static ParamGenerator&
GetInstance() {
static ParamGenerator instance;
return instance;
}
knowhere::Config Gen(const milvus::engine::IndexType& type) {
knowhere::Config
Gen(const milvus::engine::IndexType& type) {
switch (type) {
case milvus::engine::IndexType::FAISS_IDMAP: {
auto tempconf = std::make_shared<knowhere::Cfg>();
@ -114,36 +113,34 @@ class ParamGenerator {
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
// case milvus::engine::IndexType::FAISS_IVFPQ_CPU:
// case milvus::engine::IndexType::FAISS_IVFPQ_GPU: {
// auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
// tempconf->nlist = 100;
// tempconf->nprobe = 16;
// tempconf->nbits = 8;
// tempconf->m = 8;
// tempconf->metric_type = knowhere::METRICTYPE::L2;
// return tempconf;
// }
// case milvus::engine::IndexType::NSG_MIX: {
// auto tempconf = std::make_shared<knowhere::NSGCfg>();
// tempconf->nlist = 100;
// tempconf->nprobe = 16;
// tempconf->search_length = 8;
// tempconf->knng = 200;
// tempconf->search_length = 40; // TODO(linxj): be 20 when search
// tempconf->out_degree = 60;
// tempconf->candidate_pool_size = 200;
// tempconf->metric_type = knowhere::METRICTYPE::L2;
// return tempconf;
// }
// case milvus::engine::IndexType::FAISS_IVFPQ_CPU:
// case milvus::engine::IndexType::FAISS_IVFPQ_GPU: {
// auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
// tempconf->nlist = 100;
// tempconf->nprobe = 16;
// tempconf->nbits = 8;
// tempconf->m = 8;
// tempconf->metric_type = knowhere::METRICTYPE::L2;
// return tempconf;
// }
// case milvus::engine::IndexType::NSG_MIX: {
// auto tempconf = std::make_shared<knowhere::NSGCfg>();
// tempconf->nlist = 100;
// tempconf->nprobe = 16;
// tempconf->search_length = 8;
// tempconf->knng = 200;
// tempconf->search_length = 40; // TODO(linxj): be 20 when search
// tempconf->out_degree = 60;
// tempconf->candidate_pool_size = 200;
// tempconf->metric_type = knowhere::METRICTYPE::L2;
// return tempconf;
// }
}
}
};
//class SanityCheck : public DataGenBase {
// class SanityCheck : public DataGenBase {
// public:
// void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, long *ids,
// const int &k, long *gt_ids, float *gt_dis) override;
//};