mirror of https://github.com/milvus-io/milvus.git
Merge pull request #240 from youny626/0.6.0
Change default compile option to CPU version & add MKL/OpenBlas message at server start & update config and unittestpull/245/head
Jin Hai
GitHub
commit
3eda414b89
|
|
@ -31,16 +31,16 @@ GET_CURRENT_TIME(BUILD_TIME)
|
|||
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
|
||||
message(STATUS "Build time = ${BUILD_TIME}")
|
||||
|
||||
MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
|
||||
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
|
||||
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
||||
if(GIT_BRANCH_NAME STREQUAL "")
|
||||
if (GIT_BRANCH_NAME STREQUAL "")
|
||||
execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
||||
endif()
|
||||
ENDMACRO (GET_GIT_BRANCH_NAME)
|
||||
endif ()
|
||||
ENDMACRO(GET_GIT_BRANCH_NAME)
|
||||
|
||||
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
|
||||
message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
|
||||
if(NOT GIT_BRANCH_NAME STREQUAL "")
|
||||
if (NOT GIT_BRANCH_NAME STREQUAL "")
|
||||
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
|
||||
endif ()
|
||||
|
||||
|
|
@ -69,7 +69,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
|
|||
OR MILVUS_VERSION_PATCH STREQUAL "")
|
||||
message(WARNING "Failed to determine Milvus version from git branch name")
|
||||
set(MILVUS_VERSION "0.6.0")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
message(STATUS "Build version = ${MILVUS_VERSION}")
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY)
|
||||
|
|
@ -118,17 +118,17 @@ include(DefineOptions)
|
|||
include(BuildUtils)
|
||||
include(ThirdPartyPackages)
|
||||
|
||||
set(MILVUS_GPU_VERSION false)
|
||||
if (MILVUS_CPU_VERSION)
|
||||
message(STATUS "Building Milvus CPU version")
|
||||
add_compile_definitions("MILVUS_CPU_VERSION")
|
||||
else ()
|
||||
set(MILVUS_CPU_VERSION false)
|
||||
if (MILVUS_GPU_VERSION)
|
||||
message(STATUS "Building Milvus GPU version")
|
||||
set(MILVUS_GPU_VERSION true)
|
||||
add_compile_definitions("MILVUS_GPU_VERSION")
|
||||
enable_language(CUDA)
|
||||
find_package(CUDA 10 REQUIRED)
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
|
||||
else ()
|
||||
message(STATUS "Building Milvus CPU version")
|
||||
set(MILVUS_CPU_VERSION true)
|
||||
add_compile_definitions("MILVUS_CPU_VERSION")
|
||||
endif ()
|
||||
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ USE_JFROG_CACHE="OFF"
|
|||
RUN_CPPLINT="OFF"
|
||||
CUSTOMIZATION="OFF" # default use ori faiss
|
||||
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
CPU_VERSION="OFF"
|
||||
GPU_VERSION="OFF" #defaults to CPU version
|
||||
WITH_MKL="OFF"
|
||||
|
||||
CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
|
||||
|
|
@ -51,7 +51,7 @@ do
|
|||
c)
|
||||
BUILD_COVERAGE="ON"
|
||||
;;
|
||||
g)
|
||||
z)
|
||||
PROFILING="ON"
|
||||
;;
|
||||
j)
|
||||
|
|
@ -60,8 +60,8 @@ do
|
|||
x)
|
||||
CUSTOMIZATION="OFF" # force use ori faiss
|
||||
;;
|
||||
z)
|
||||
CPU_VERSION="ON"
|
||||
g)
|
||||
GPU_VERSION="ON"
|
||||
;;
|
||||
m)
|
||||
WITH_MKL="ON"
|
||||
|
|
@ -77,14 +77,14 @@ parameter:
|
|||
-l: run cpplint, clang-format and clang-tidy(default: OFF)
|
||||
-r: remove previous build directory(default: OFF)
|
||||
-c: code coverage(default: OFF)
|
||||
-g: profiling(default: OFF)
|
||||
-z: profiling(default: OFF)
|
||||
-j: use jfrog cache build directory(default: OFF)
|
||||
-z: build pure CPU version(default: OFF)
|
||||
-g: build GPU version(default: OFF)
|
||||
-m: build with MKL(default: OFF)
|
||||
-h: help
|
||||
|
||||
usage:
|
||||
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-z] [-m] [-h]
|
||||
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
|
||||
"
|
||||
exit 0
|
||||
;;
|
||||
|
|
@ -116,7 +116,7 @@ CMAKE_CMD="cmake \
|
|||
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
|
||||
-DCUSTOMIZATION=${CUSTOMIZATION} \
|
||||
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
|
||||
-DMILVUS_CPU_VERSION=${CPU_VERSION} \
|
||||
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
|
||||
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
|
||||
../"
|
||||
echo ${CMAKE_CMD}
|
||||
|
|
|
|||
|
|
@ -13,16 +13,16 @@ macro(define_option name description default)
|
|||
endmacro()
|
||||
|
||||
function(list_join lst glue out)
|
||||
if("${${lst}}" STREQUAL "")
|
||||
if ("${${lst}}" STREQUAL "")
|
||||
set(${out} "" PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
list(GET ${lst} 0 joined)
|
||||
list(REMOVE_AT ${lst} 0)
|
||||
foreach(item ${${lst}})
|
||||
foreach (item ${${lst}})
|
||||
set(joined "${joined}${glue}${item}")
|
||||
endforeach()
|
||||
endforeach ()
|
||||
set(${out} ${joined} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
|
|
@ -35,15 +35,15 @@ macro(define_option_string name description default)
|
|||
|
||||
set("${name}_OPTION_ENUM" ${ARGN})
|
||||
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
|
||||
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
|
||||
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
|
||||
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
|
||||
endif()
|
||||
endif ()
|
||||
endmacro()
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
set_option_category("CPU version")
|
||||
set_option_category("GPU version")
|
||||
|
||||
define_option(MILVUS_CPU_VERSION "Build CPU version only" OFF)
|
||||
define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
set_option_category("Thirdparty")
|
||||
|
|
@ -51,11 +51,11 @@ set_option_category("Thirdparty")
|
|||
set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO")
|
||||
|
||||
define_option_string(MILVUS_DEPENDENCY_SOURCE
|
||||
"Method to use for acquiring MILVUS's build dependencies"
|
||||
"${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
|
||||
"AUTO"
|
||||
"BUNDLED"
|
||||
"SYSTEM")
|
||||
"Method to use for acquiring MILVUS's build dependencies"
|
||||
"${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
|
||||
"AUTO"
|
||||
"BUNDLED"
|
||||
"SYSTEM")
|
||||
|
||||
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
|
||||
"Show output from ExternalProjects rather than just logging to files" ON)
|
||||
|
|
@ -75,33 +75,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
|
|||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
|
||||
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
|
||||
|
||||
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
if(MSVC)
|
||||
set_option_category("MSVC")
|
||||
|
||||
define_option(MSVC_LINK_VERBOSE
|
||||
"Pass verbose linking options when linking libraries and executables"
|
||||
OFF)
|
||||
|
||||
define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF)
|
||||
endif()
|
||||
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
set_option_category("Test and benchmark")
|
||||
|
||||
unset(MILVUS_BUILD_TESTS CACHE)
|
||||
if (BUILD_UNIT_TEST)
|
||||
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
|
||||
else()
|
||||
else ()
|
||||
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
|
||||
endif(BUILD_UNIT_TEST)
|
||||
endif (BUILD_UNIT_TEST)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
macro(config_summary)
|
||||
|
|
@ -113,12 +101,12 @@ macro(config_summary)
|
|||
message(STATUS " Generator: ${CMAKE_GENERATOR}")
|
||||
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
|
||||
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
|
||||
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
|
||||
message(
|
||||
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
foreach(category ${MILVUS_OPTION_CATEGORIES})
|
||||
foreach (category ${MILVUS_OPTION_CATEGORIES})
|
||||
|
||||
message(STATUS)
|
||||
message(STATUS "${category} options:")
|
||||
|
|
@ -126,50 +114,50 @@ macro(config_summary)
|
|||
set(option_names ${MILVUS_${category}_OPTION_NAMES})
|
||||
|
||||
set(max_value_length 0)
|
||||
foreach(name ${option_names})
|
||||
foreach (name ${option_names})
|
||||
string(LENGTH "\"${${name}}\"" value_length)
|
||||
if(${max_value_length} LESS ${value_length})
|
||||
if (${max_value_length} LESS ${value_length})
|
||||
set(max_value_length ${value_length})
|
||||
endif()
|
||||
endforeach()
|
||||
endif ()
|
||||
endforeach ()
|
||||
|
||||
foreach(name ${option_names})
|
||||
if("${${name}_OPTION_TYPE}" STREQUAL "string")
|
||||
foreach (name ${option_names})
|
||||
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
|
||||
set(value "\"${${name}}\"")
|
||||
else()
|
||||
else ()
|
||||
set(value "${${name}}")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
set(default ${${name}_OPTION_DEFAULT})
|
||||
set(description ${${name}_OPTION_DESCRIPTION})
|
||||
string(LENGTH ${description} description_length)
|
||||
if(${description_length} LESS 70)
|
||||
if (${description_length} LESS 70)
|
||||
string(
|
||||
SUBSTRING
|
||||
" "
|
||||
${description_length} -1 description_padding)
|
||||
else()
|
||||
else ()
|
||||
set(description_padding "
|
||||
")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
set(comment "[${name}]")
|
||||
|
||||
if("${value}" STREQUAL "${default}")
|
||||
if ("${value}" STREQUAL "${default}")
|
||||
set(comment "[default] ${comment}")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
|
||||
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
|
||||
set(comment "${comment} [${${name}_OPTION_ENUM}]")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
string(
|
||||
SUBSTRING "${value} "
|
||||
0 ${max_value_length} value)
|
||||
|
||||
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
|
||||
endforeach()
|
||||
endforeach ()
|
||||
|
||||
endforeach()
|
||||
endforeach ()
|
||||
|
||||
endmacro()
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ cache_config:
|
|||
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
|
||||
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
|
||||
|
||||
# Skip the following config if you are using GPU version
|
||||
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
|
||||
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
|
||||
#Uncomment the following config if you are using GPU version
|
||||
# gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
|
||||
# gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
|
||||
|
||||
engine_config:
|
||||
use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
|
||||
|
|
@ -44,5 +44,4 @@ engine_config:
|
|||
resource_config:
|
||||
search_resources: # define the devices used for search computation, must be in format: cpu or gpux
|
||||
- cpu
|
||||
- gpu0
|
||||
index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu / gpux
|
||||
index_build_device: cpu # CPU / GPU used for building index, must be in format: cpu / gpux
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
|
|||
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
|
||||
|
||||
add_subdirectory(index)
|
||||
if (BUILD_FAISS_WITH_MKL)
|
||||
add_compile_definitions("WITH_MKL")
|
||||
endif ()
|
||||
|
||||
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
foreach (dir ${INDEX_INCLUDE_DIRS})
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ cmake_minimum_required(VERSION 3.14)
|
|||
message(STATUS "------------------------------KNOWHERE-----------------------------------")
|
||||
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
|
||||
|
||||
set(KNOWHERE_VERSION "0.5.0")
|
||||
set(KNOWHERE_VERSION "0.6.0")
|
||||
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}")
|
||||
project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX)
|
||||
set(CMAKE_CXX_STANDARD 14)
|
||||
|
|
@ -72,17 +72,17 @@ include(ExternalProject)
|
|||
include(DefineOptionsCore)
|
||||
include(BuildUtilsCore)
|
||||
|
||||
set(KNOWHERE_GPU_VERSION false)
|
||||
if (MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION)
|
||||
message(STATUS "Building Knowhere CPU version")
|
||||
add_compile_definitions("MILVUS_CPU_VERSION")
|
||||
else ()
|
||||
set(KNOWHERE_CPU_VERSION false)
|
||||
if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
|
||||
message(STATUS "Building Knowhere GPU version")
|
||||
add_compile_definitions("MILVUS_GPU_VERSION")
|
||||
set(KNOWHERE_GPU_VERSION true)
|
||||
enable_language(CUDA)
|
||||
find_package(CUDA 10 REQUIRED)
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
|
||||
else ()
|
||||
message(STATUS "Building Knowhere CPU version")
|
||||
set(KNOWHERE_CPU_VERSION true)
|
||||
add_compile_definitions("MILVUS_CPU_VERSION")
|
||||
endif ()
|
||||
|
||||
include(ThirdPartyPackagesCore)
|
||||
|
|
|
|||
|
|
@ -41,12 +41,12 @@ macro(define_option_string name description default)
|
|||
endmacro()
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
set_option_category("CPU version")
|
||||
set_option_category("GPU version")
|
||||
|
||||
if (MILVUS_CPU_VERSION)
|
||||
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
|
||||
if (MILVUS_GPU_VERSION)
|
||||
define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON)
|
||||
else ()
|
||||
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
|
||||
define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
|
||||
endif ()
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
|
@ -81,17 +81,6 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
|
|||
|
||||
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
if (MSVC)
|
||||
set_option_category("MSVC")
|
||||
|
||||
define_option(MSVC_LINK_VERBOSE
|
||||
"Pass verbose linking options when linking libraries and executables"
|
||||
OFF)
|
||||
|
||||
define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
|
||||
endif ()
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
set_option_category("Test and benchmark")
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,13 @@ print_banner() {
|
|||
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
|
||||
std::cout << std::endl;
|
||||
std::cout << "Welcome to Milvus!" << std::endl;
|
||||
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl;
|
||||
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with "
|
||||
#ifdef WITH_MKL
|
||||
<< "MKL"
|
||||
#else
|
||||
<< "OpenBLAS"
|
||||
#endif
|
||||
<< " library." << std::endl;
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
std::cout << "You are using Milvus CPU version" << std::endl;
|
||||
#else
|
||||
|
|
@ -137,7 +143,7 @@ main(int argc, char* argv[]) {
|
|||
|
||||
s = server.Start();
|
||||
if (s.ok()) {
|
||||
std::cout << "Milvus server start successfully." << std::endl;
|
||||
std::cout << "Milvus server started successfully!" << std::endl;
|
||||
} else {
|
||||
goto FAIL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
#include "config/YamlConfigMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/StringHelpFunctions.h"
|
||||
#include "utils/ValidationUtil.h"
|
||||
|
||||
namespace milvus {
|
||||
|
|
@ -343,6 +344,11 @@ Config::ResetDefaultConfig() {
|
|||
return s;
|
||||
}
|
||||
|
||||
s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
|
|
@ -403,8 +409,7 @@ Status
|
|||
Config::CheckServerConfigDeployMode(const std::string& value) {
|
||||
if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") {
|
||||
return Status(SERVER_INVALID_ARGUMENT,
|
||||
"server_config.deploy_mode is not one of "
|
||||
"single, cluster_readonly, and cluster_writable.");
|
||||
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.");
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -592,15 +597,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
|
|||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
} else {
|
||||
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
|
||||
int gpu_index;
|
||||
Status s = GetResourceConfigIndexBuildDevice(gpu_index);
|
||||
int device_id;
|
||||
Status s = GetResourceConfigIndexBuildDevice(device_id);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
size_t gpu_memory;
|
||||
if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) {
|
||||
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index);
|
||||
if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) {
|
||||
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id);
|
||||
return Status(SERVER_UNEXPECTED_ERROR, msg);
|
||||
} else if (gpu_cache_capacity >= gpu_memory) {
|
||||
std::string msg = "Invalid gpu cache capacity: " + value +
|
||||
|
|
@ -689,29 +694,33 @@ Config::CheckResourceConfigMode(const std::string& value) {
|
|||
}
|
||||
|
||||
Status
|
||||
CheckGpuDevice(const std::string& value) {
|
||||
CheckResource(const std::string& value) {
|
||||
std::string s = value;
|
||||
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
|
||||
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
if (s != "cpu") {
|
||||
return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s);
|
||||
}
|
||||
#else
|
||||
const std::regex pat("gpu(\\d+)");
|
||||
std::cmatch m;
|
||||
if (!std::regex_match(value.c_str(), m, pat)) {
|
||||
std::string msg = "Invalid gpu device: " + value +
|
||||
". Possible reason: resource_config.search_resources does not match your hardware.";
|
||||
const std::regex pat("cpu|gpu(\\d+)");
|
||||
std::smatch m;
|
||||
if (!std::regex_match(s, m, pat)) {
|
||||
std::string msg = "Invalid search resource: " + value +
|
||||
". Possible reason: resource_config.search_resources is not in the format of cpux or gpux";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
|
||||
int32_t gpu_index = std::stoi(value.substr(3));
|
||||
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
|
||||
std::string msg = "Invalid gpu device: " + value +
|
||||
". Possible reason: resource_config.search_resources does not match your hardware.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
if (s.compare(0, 3, "gpu") == 0) {
|
||||
int32_t gpu_index = std::stoi(s.substr(3));
|
||||
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
|
||||
std::string msg = "Invalid search resource: " + value +
|
||||
". Possible reason: resource_config.search_resources does not match your hardware.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -724,38 +733,20 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
|
|||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
|
||||
bool cpu_found = false, gpu_found = false;
|
||||
for (auto& device : value) {
|
||||
if (device == "cpu") {
|
||||
cpu_found = true;
|
||||
continue;
|
||||
for (auto& resource : value) {
|
||||
auto status = CheckResource(resource);
|
||||
if (!status.ok()) {
|
||||
return Status(SERVER_INVALID_ARGUMENT, status.message());
|
||||
}
|
||||
if (CheckGpuDevice(device).ok()) {
|
||||
gpu_found = true;
|
||||
} else {
|
||||
std::string msg = "Invalid search resource: " + device +
|
||||
". Possible reason: resource_config.search_resources does not match your hardware.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
}
|
||||
|
||||
if (cpu_found && !gpu_found) {
|
||||
std::string msg =
|
||||
"Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status
|
||||
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
|
||||
// if (value == "cpu") {
|
||||
// return Status::OK();
|
||||
// }
|
||||
if (!CheckGpuDevice(value).ok()) {
|
||||
std::string msg = "Invalid index build device: " + value +
|
||||
". Possible reason: resource_config.index_build_device does not match your hardware.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
auto status = CheckResource(value);
|
||||
if (!status.ok()) {
|
||||
return Status(SERVER_INVALID_ARGUMENT, status.message());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -796,6 +787,22 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key
|
|||
return value;
|
||||
}
|
||||
|
||||
std::string
|
||||
Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim,
|
||||
const std::string& default_value) {
|
||||
std::string value;
|
||||
if (!GetConfigValueInMem(parent_key, child_key, value).ok()) {
|
||||
std::vector<std::string> sequence = GetConfigNode(parent_key).GetSequence(child_key);
|
||||
if (sequence.empty()) {
|
||||
value = default_value;
|
||||
} else {
|
||||
server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value);
|
||||
}
|
||||
SetConfigValueInMem(parent_key, child_key, value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
Status
|
||||
Config::GetServerConfigAddress(std::string& value) {
|
||||
value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
|
||||
|
|
@ -1019,8 +1026,10 @@ Config::GetResourceConfigMode(std::string& value) {
|
|||
|
||||
Status
|
||||
Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
|
||||
ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
|
||||
value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
|
||||
std::string str =
|
||||
GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES,
|
||||
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
|
||||
server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value);
|
||||
return CheckResourceConfigSearchResources(value);
|
||||
}
|
||||
|
||||
|
|
@ -1033,10 +1042,10 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
if (str != "cpu") {
|
||||
value = std::stoi(str.substr(3));
|
||||
if (str == "cpu") {
|
||||
value = CPU_DEVICE_ID;
|
||||
} else {
|
||||
value = -1;
|
||||
value = std::stoi(str.substr(3));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
@ -1163,7 +1172,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
|
||||
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1174,7 +1183,7 @@ Config::SetMetricConfigCollector(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);
|
||||
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1185,7 +1194,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);
|
||||
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1197,7 +1206,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
|
||||
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1208,7 +1217,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
|
||||
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1219,7 +1228,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
|
||||
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1230,7 +1239,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
|
||||
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1241,7 +1250,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);
|
||||
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1253,7 +1262,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
|
||||
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1264,7 +1273,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);
|
||||
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1275,7 +1284,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
|
||||
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1287,7 +1296,21 @@ Config::SetResourceConfigMode(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);
|
||||
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status
|
||||
Config::SetResourceConfigSearchResources(const std::string& value) {
|
||||
std::vector<std::string> res_vec;
|
||||
server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec);
|
||||
|
||||
Status s = CheckResourceConfigSearchResources(res_vec);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1298,7 +1321,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
|
|||
return s;
|
||||
}
|
||||
|
||||
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
|
||||
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -92,12 +92,19 @@ static const char* CONFIG_RESOURCE = "resource_config";
|
|||
static const char* CONFIG_RESOURCE_MODE = "mode";
|
||||
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
|
||||
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
|
||||
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu";
|
||||
#else
|
||||
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";
|
||||
#endif
|
||||
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu";
|
||||
#else
|
||||
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
|
||||
#endif
|
||||
const int32_t CPU_DEVICE_ID = -1;
|
||||
|
||||
class Config {
|
||||
public:
|
||||
|
|
@ -185,6 +192,9 @@ class Config {
|
|||
|
||||
std::string
|
||||
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
|
||||
std::string
|
||||
GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
|
||||
const std::string& default_value = "");
|
||||
|
||||
public:
|
||||
/* server config */
|
||||
|
|
@ -306,6 +316,8 @@ class Config {
|
|||
Status
|
||||
SetResourceConfigMode(const std::string& value);
|
||||
Status
|
||||
SetResourceConfigSearchResources(const std::string& value);
|
||||
Status
|
||||
SetResourceConfigIndexBuildDevice(const std::string& value);
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou
|
|||
}
|
||||
}
|
||||
|
||||
Status
|
||||
void
|
||||
StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter,
|
||||
std::vector<std::string>& result) {
|
||||
if (str.empty()) {
|
||||
return Status::OK();
|
||||
return;
|
||||
}
|
||||
|
||||
size_t last = 0;
|
||||
size_t index = str.find_first_of(delimeter, last);
|
||||
while (index != std::string::npos) {
|
||||
result.emplace_back(str.substr(last, index - last));
|
||||
last = index + 1;
|
||||
index = str.find_first_of(delimeter, last);
|
||||
size_t prev = 0, pos = 0;
|
||||
while (true) {
|
||||
pos = str.find_first_of(delimeter, prev);
|
||||
if (pos == std::string::npos) {
|
||||
result.emplace_back(str.substr(prev));
|
||||
break;
|
||||
} else {
|
||||
result.emplace_back(str.substr(prev, pos - prev));
|
||||
prev = pos + 1;
|
||||
}
|
||||
}
|
||||
if (index - last > 0) {
|
||||
std::string temp = str.substr(last);
|
||||
result.emplace_back(temp);
|
||||
}
|
||||
|
||||
void
|
||||
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter,
|
||||
std::string& result) {
|
||||
if (strs.empty()) {
|
||||
result = "";
|
||||
return;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
result = strs[0];
|
||||
for (size_t i = 1; i < strs.size(); i++) {
|
||||
result = result + delimeter + strs[i];
|
||||
}
|
||||
}
|
||||
|
||||
Status
|
||||
StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,
|
||||
std::vector<std::string>& result) {
|
||||
if (quote.empty()) {
|
||||
return SplitStringByDelimeter(str, delimeter, result);
|
||||
SplitStringByDelimeter(str, delimeter, result);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
size_t last = 0;
|
||||
size_t index = str.find_first_of(quote, last);
|
||||
if (index == std::string::npos) {
|
||||
return SplitStringByDelimeter(str, delimeter, result);
|
||||
SplitStringByDelimeter(str, delimeter, result);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::string process_str = str;
|
||||
|
|
@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
|
|||
}
|
||||
|
||||
if (!process_str.empty()) {
|
||||
return SplitStringByDelimeter(process_str, delimeter, result);
|
||||
SplitStringByDelimeter(process_str, delimeter, result);
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
|||
|
|
@ -43,9 +43,12 @@ class StringHelpFunctions {
|
|||
// ,b, | b |
|
||||
// ,, | |
|
||||
// a a
|
||||
static Status
|
||||
static void
|
||||
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
|
||||
|
||||
static void
|
||||
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
|
||||
|
||||
// assume the table has two columns, quote='\"', delimeter=','
|
||||
// a,b a | b
|
||||
// "aa,gg,yy",b aa,gg,yy | b
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ set(helper_files
|
|||
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/Status.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
|
||||
${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include "server/Config.h"
|
||||
#include "server/utils.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/StringHelpFunctions.h"
|
||||
#include "utils/ValidationUtil.h"
|
||||
|
||||
namespace {
|
||||
|
|
@ -98,6 +99,326 @@ TEST_F(ConfigTest, CONFIG_TEST) {
|
|||
ASSERT_TRUE(seqs.empty());
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
|
||||
std::string config_path(CONFIG_PATH);
|
||||
milvus::server::Config& config = milvus::server::Config::GetInstance();
|
||||
milvus::Status s;
|
||||
std::string str_val;
|
||||
int32_t int32_val;
|
||||
int64_t int64_val;
|
||||
float float_val;
|
||||
bool bool_val;
|
||||
|
||||
/* server config */
|
||||
std::string server_addr = "192.168.1.155";
|
||||
s = config.SetServerConfigAddress(server_addr);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetServerConfigAddress(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == server_addr);
|
||||
|
||||
std::string server_port = "12345";
|
||||
s = config.SetServerConfigPort(server_port);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetServerConfigPort(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == server_port);
|
||||
|
||||
std::string server_mode = "cluster_readonly";
|
||||
s = config.SetServerConfigDeployMode(server_mode);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetServerConfigDeployMode(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == server_mode);
|
||||
|
||||
std::string server_time_zone = "UTC+6";
|
||||
s = config.SetServerConfigTimeZone(server_time_zone);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetServerConfigTimeZone(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == server_time_zone);
|
||||
|
||||
/* db config */
|
||||
std::string db_primary_path = "/home/zilliz";
|
||||
s = config.SetDBConfigPrimaryPath(db_primary_path);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigPrimaryPath(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == db_primary_path);
|
||||
|
||||
std::string db_secondary_path = "/home/zilliz";
|
||||
s = config.SetDBConfigSecondaryPath(db_secondary_path);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigSecondaryPath(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == db_secondary_path);
|
||||
|
||||
std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus";
|
||||
s = config.SetDBConfigBackendUrl(db_backend_url);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigBackendUrl(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == db_backend_url);
|
||||
|
||||
int32_t db_archive_disk_threshold = 100;
|
||||
s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigArchiveDiskThreshold(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == db_archive_disk_threshold);
|
||||
|
||||
int32_t db_archive_days_threshold = 365;
|
||||
s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigArchiveDaysThreshold(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == db_archive_days_threshold);
|
||||
|
||||
int32_t db_insert_buffer_size = 2;
|
||||
s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetDBConfigInsertBufferSize(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == db_insert_buffer_size);
|
||||
|
||||
/* metric config */
|
||||
bool metric_enable_monitor = false;
|
||||
s = config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetMetricConfigEnableMonitor(bool_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(bool_val == metric_enable_monitor);
|
||||
|
||||
std::string metric_collector = "prometheus";
|
||||
s = config.SetMetricConfigCollector(metric_collector);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetMetricConfigCollector(str_val);
|
||||
ASSERT_TRUE(str_val == metric_collector);
|
||||
|
||||
std::string metric_prometheus_port = "2222";
|
||||
s = config.SetMetricConfigPrometheusPort(metric_prometheus_port);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetMetricConfigPrometheusPort(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == metric_prometheus_port);
|
||||
|
||||
/* cache config */
|
||||
int64_t cache_cpu_cache_capacity = 5;
|
||||
s = config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetCacheConfigCpuCacheCapacity(int64_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int64_val == cache_cpu_cache_capacity);
|
||||
|
||||
float cache_cpu_cache_threshold = 0.1;
|
||||
s = config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetCacheConfigCpuCacheThreshold(float_val);
|
||||
ASSERT_TRUE(float_val == cache_cpu_cache_threshold);
|
||||
|
||||
int64_t cache_gpu_cache_capacity = 1;
|
||||
s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetCacheConfigGpuCacheCapacity(int64_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int64_val == cache_gpu_cache_capacity);
|
||||
|
||||
float cache_gpu_cache_threshold = 0.2;
|
||||
s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetCacheConfigGpuCacheThreshold(float_val);
|
||||
ASSERT_TRUE(float_val == cache_gpu_cache_threshold);
|
||||
|
||||
bool cache_insert_data = true;
|
||||
s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetCacheConfigCacheInsertData(bool_val);
|
||||
ASSERT_TRUE(bool_val == cache_insert_data);
|
||||
|
||||
/* engine config */
|
||||
int32_t engine_use_blas_threshold = 50;
|
||||
s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetEngineConfigUseBlasThreshold(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == engine_use_blas_threshold);
|
||||
|
||||
int32_t engine_omp_thread_num = 8;
|
||||
s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetEngineConfigOmpThreadNum(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == engine_omp_thread_num);
|
||||
|
||||
int32_t engine_gpu_search_threshold = 800;
|
||||
s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetEngineConfigGpuSearchThreshold(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == engine_gpu_search_threshold);
|
||||
|
||||
/* resource config */
|
||||
std::string resource_mode = "simple";
|
||||
s = config.SetResourceConfigMode(resource_mode);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetResourceConfigMode(str_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(str_val == resource_mode);
|
||||
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
std::vector<std::string> search_resources = {"cpu"};
|
||||
#else
|
||||
std::vector<std::string> search_resources = {"cpu", "gpu0"};
|
||||
#endif
|
||||
std::vector<std::string> res_vec;
|
||||
std::string res_str;
|
||||
milvus::server::StringHelpFunctions::MergeStringWithDelimeter(
|
||||
search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str);
|
||||
s = config.SetResourceConfigSearchResources(res_str);
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetResourceConfigSearchResources(res_vec);
|
||||
ASSERT_TRUE(s.ok());
|
||||
for (size_t i = 0; i < search_resources.size(); i++) {
|
||||
ASSERT_TRUE(search_resources[i] == res_vec[i]);
|
||||
}
|
||||
|
||||
#ifdef MILVUS_CPU_VERSION
|
||||
int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID;
|
||||
s = config.SetResourceConfigIndexBuildDevice("cpu");
|
||||
#else
|
||||
int32_t resource_index_build_device = 0;
|
||||
s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device));
|
||||
#endif
|
||||
ASSERT_TRUE(s.ok());
|
||||
s = config.GetResourceConfigIndexBuildDevice(int32_val);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_TRUE(int32_val == resource_index_build_device);
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
|
||||
std::string config_path(CONFIG_PATH);
|
||||
milvus::server::Config& config = milvus::server::Config::GetInstance();
|
||||
milvus::Status s;
|
||||
|
||||
s = config.LoadConfigFile("");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.LoadConfigFile(config_path + INVALID_CONFIG_FILE);
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.LoadConfigFile(config_path + "dummy.yaml");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* server config */
|
||||
s = config.SetServerConfigAddress("0.0.0");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetServerConfigAddress("0.0.0.256");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetServerConfigPort("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetServerConfigPort("99999");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetServerConfigDeployMode("cluster");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetServerConfigTimeZone("GM");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetServerConfigTimeZone("GMT8");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetServerConfigTimeZone("UTCA");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* db config */
|
||||
s = config.SetDBConfigPrimaryPath("");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
// s = config.SetDBConfigSecondaryPath("");
|
||||
// ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetDBConfigBackendUrl("http://www.google.com");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetDBConfigBackendUrl("sqlite://:@:");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetDBConfigArchiveDiskThreshold("0x10");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetDBConfigArchiveDaysThreshold("0x10");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetDBConfigInsertBufferSize("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetDBConfigInsertBufferSize("0");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetDBConfigInsertBufferSize("2048");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* metric config */
|
||||
s = config.SetMetricConfigEnableMonitor("Y");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetMetricConfigCollector("zilliz");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetMetricConfigPrometheusPort("0xff");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* cache config */
|
||||
s = config.SetCacheConfigCpuCacheCapacity("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetCacheConfigCpuCacheCapacity("0");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetCacheConfigCpuCacheCapacity("2048");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetCacheConfigCpuCacheThreshold("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetCacheConfigCpuCacheThreshold("1.0");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
s = config.SetCacheConfigGpuCacheCapacity("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetCacheConfigGpuCacheCapacity("128");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetCacheConfigGpuCacheThreshold("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetCacheConfigGpuCacheThreshold("1.0");
|
||||
ASSERT_FALSE(s.ok());
|
||||
#endif
|
||||
|
||||
s = config.SetCacheConfigCacheInsertData("N");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* engine config */
|
||||
s = config.SetEngineConfigUseBlasThreshold("0xff");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetEngineConfigOmpThreadNum("a");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetEngineConfigOmpThreadNum("10000");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetEngineConfigGpuSearchThreshold("-1");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
/* resource config */
|
||||
s = config.SetResourceConfigMode("default");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetResourceConfigSearchResources("gpu10");
|
||||
ASSERT_FALSE(s.ok());
|
||||
|
||||
s = config.SetResourceConfigIndexBuildDevice("gup2");
|
||||
ASSERT_FALSE(s.ok());
|
||||
s = config.SetResourceConfigIndexBuildDevice("gpu16");
|
||||
ASSERT_FALSE(s.ok());
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
|
||||
std::string config_path(CONFIG_PATH);
|
||||
milvus::server::Config& config = milvus::server::Config::GetInstance();
|
||||
|
|
|
|||
|
|
@ -117,12 +117,11 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
|
|||
|
||||
str = "a,b,c";
|
||||
std::vector<std::string> result;
|
||||
auto status = milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
|
||||
ASSERT_TRUE(status.ok());
|
||||
milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
|
||||
ASSERT_EQ(result.size(), 3UL);
|
||||
|
||||
result.clear();
|
||||
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
||||
auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
||||
ASSERT_TRUE(status.ok());
|
||||
ASSERT_EQ(result.size(), 3UL);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue