milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into '0.5.0' 

#48 Config unittest failed

See merge request megasearch/milvus!770

Former-commit-id: 9f74cc2b01e09deee21a0d2fac4d7f43ffdb83d6
pull/191/head
peng.xu 2019-10-19 17:09:28 +08:00
parent 73e7478d30 ff27d56571
commit d51b0ca2e2
8 changed files with 135 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -26,13 +26,14 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-653 - When config check fail, Milvus close without message
- MS-654 - Describe index timeout when building index
- MS-658 - Fix SQ8 Hybrid can't search
- \#9 Change default gpu_cache_capacity to 4
- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
- \#9 - Change default gpu_cache_capacity to 4
- \#20 - C++ sdk example get grpc error
- \#23 - Add unittest to improve code coverage
- \#31 - make clang-format failed after run build.sh -l
- \#39 - Create SQ8H index hang if using github server version
- \#30 - Some troubleshoot messages in Milvus do not provide enough information
- \#48 - Config unittest failed
## Improvement
- MS-552 - Add and change the easylogging library

8
core/unittest/server/CMakeLists.txt
View File

@ -67,11 +67,3 @@ target_link_libraries(test_server
)
install(TARGETS test_server DESTINATION unittest)
configure_file(appendix/server_config.yaml
"${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml"
COPYONLY)
configure_file(appendix/log_config.conf
"${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf"
COPYONLY)

27
core/unittest/server/appendix/log_config.conf
View File

@ -1,27 +0,0 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = false
SUBSECOND_PRECISION = 3
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB
* DEBUG:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
* TRACE:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
TO_STANDARD_OUTPUT = false
## Error logs
* ERROR:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
* FATAL:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"

43
core/unittest/server/appendix/server_config.yaml
View File

@ -1,43 +0,0 @@
# Default values are used when you make no changes to the following parameters.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534
deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8
db_config:
primary_path: /tmp/milvus # path used to store data and meta
secondary_path: # path used to store data only, split by semicolon
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus uses to fetch metrics
cache_config:
cpu_cache_capacity: 16 # GB, CPU memory used for cache
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity: 4 # GB, GPU memory used for cache
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
cache_insert_data: false # whether to load inserted data into cache
engine_config:
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
search_resources: # define the GPUs used for search computation, valid value: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index

14
core/unittest/server/test_config.cpp
View File

@ -22,28 +22,26 @@
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
#include "server/Config.h"
#include "server/utils.h"
namespace {
static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
static constexpr uint64_t KB = 1024;
static constexpr uint64_t MB = KB * 1024;
static constexpr uint64_t GB = MB * 1024;
} // namespace
TEST(ConfigTest, CONFIG_TEST) {
TEST_F(ConfigTest, CONFIG_TEST) {
milvus::server::ConfigMgr *config_mgr = milvus::server::YamlConfigMgr::GetInstance();
milvus::Status s = config_mgr->LoadConfigFile("");
ASSERT_FALSE(s.ok());
s = config_mgr->LoadConfigFile(LOG_FILE_PATH);
s = config_mgr->LoadConfigFile(INVALID_CONFIG_PATH);
ASSERT_FALSE(s.ok());
s = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
s = config_mgr->LoadConfigFile(VALID_CONFIG_PATH);
ASSERT_TRUE(s.ok());
config_mgr->Print();
@ -99,9 +97,9 @@ TEST(ConfigTest, CONFIG_TEST) {
ASSERT_TRUE(seqs.empty());
}
TEST(ConfigTest, SERVER_CONFIG_TEST) {
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
milvus::Status s = config.LoadConfigFile(VALID_CONFIG_PATH);
ASSERT_TRUE(s.ok());
s = config.ValidateConfig();

5
core/unittest/server/util_test.cpp → core/unittest/server/test_util.cpp
View File

@ -275,6 +275,11 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
#ifndef CUSTOMIZATION
if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
continue;
}
#endif
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
}
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(

91
core/unittest/server/utils.cpp Normal file
View File

@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "server/utils.h"
#include <iostream>
#include <thread>
#include <string>
#include <boost/filesystem.hpp>
namespace {
static const char
* VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
" preload_table: \n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus uses to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_cache_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_cache_threshold: 0.85 \n"
" gpu_cache_capacity: 4 # GB, GPU memory used for cache\n"
" gpu_cache_threshold: 0.85 \n"
" cache_insert_data: false # whether to load inserted data into cache\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20 \n"
"\n"
"resource_config:\n"
" search_resources: \n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
static const char* INVALID_CONFIG_STR = "*INVALID*";
void
WriteToFile(const char* file_path, const char* content) {
boost::filesystem::path fpath(file_path);
boost::filesystem::fstream fstream(fpath, std::ios_base::out);
//write data to file
fstream << content;
fstream.close();
}
} // namespace
void
ConfigTest::SetUp() {
WriteToFile(VALID_CONFIG_PATH, VALID_CONFIG_STR);
WriteToFile(INVALID_CONFIG_PATH, INVALID_CONFIG_STR);
}
void
ConfigTest::TearDown() {
boost::filesystem::remove(VALID_CONFIG_PATH);
boost::filesystem::remove(INVALID_CONFIG_PATH);
}

31
core/unittest/server/utils.h Normal file
View File

@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gtest/gtest.h>
#include <chrono>
static const char *VALID_CONFIG_PATH = "/tmp/milvus_test/valid_config.yaml";
static const char *INVALID_CONFIG_PATH = "/tmp/milvus_test/invalid_config.conf";
class ConfigTest : public ::testing::Test {
protected:
void SetUp() override;
void TearDown() override;
};