Merge branch '0.6.0' into issue-509

pull/522/head
Tinkerrr 2019-11-25 22:09:13 +08:00 committed by GitHub
commit 038850d6a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 43 additions and 25 deletions

View File

@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#458 - Index data is not compatible between 0.5 and 0.6 - \#458 - Index data is not compatible between 0.5 and 0.6
- \#486 - gpu no usage during index building - \#486 - gpu no usage during index building
- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params - \#509 - IVF_PQ index build trapped into dead loop caused by invalid params
- \#513 - Unittest DELETE_BY_RANGE sometimes failed
## Feature ## Feature
- \#12 - Pure CPU version for Milvus - \#12 - Pure CPU version for Milvus

View File

@ -7,6 +7,7 @@
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen)
[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
[中文版](README_CN.md) | [日本語版](README_JP.md) [中文版](README_CN.md) | [日本語版](README_JP.md)

View File

@ -6,6 +6,8 @@
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen)
[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
# 欢迎来到 Milvus # 欢迎来到 Milvus

View File

@ -31,9 +31,21 @@ GET_CURRENT_TIME(BUILD_TIME)
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message(STATUS "Build time = ${BUILD_TIME}") message(STATUS "Build time = ${BUILD_TIME}")
if (NOT DEFINED CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.")
endif ()
set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]")
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif ()
if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif ()
ENDMACRO(GET_GIT_BRANCH_NAME) ENDMACRO(GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
@ -43,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "")
endif () endif ()
set(MILVUS_VERSION "${GIT_BRANCH_NAME}") set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}")
if (CMAKE_BUILD_TYPE STREQUAL "Release") if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "Release") set(BUILD_TYPE "Release")

View File

@ -105,7 +105,8 @@ DBImpl::Stop() {
shutting_down_.store(true, std::memory_order_release); shutting_down_.store(true, std::memory_order_release);
// makesure all memory data serialized // makesure all memory data serialized
MemSerialize(); std::set<std::string> sync_table_ids;
SyncMemData(sync_table_ids);
// wait compaction/buildindex finish // wait compaction/buildindex finish
bg_timer_thread_.join(); bg_timer_thread_.join();
@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
return SHUTDOWN_ERROR; return SHUTDOWN_ERROR;
} }
Status status; // serialize memory data
std::set<std::string> sync_table_ids;
auto status = SyncMemData(sync_table_ids);
{ {
std::unique_lock<std::mutex> lock(build_index_mutex_); std::unique_lock<std::mutex> lock(build_index_mutex_);
@ -588,12 +592,12 @@ DBImpl::StartMetricTask() {
} }
Status Status
DBImpl::MemSerialize() { DBImpl::SyncMemData(std::set<std::string>& sync_table_ids) {
std::lock_guard<std::mutex> lck(mem_serialize_mutex_); std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
std::set<std::string> temp_table_ids; std::set<std::string> temp_table_ids;
mem_mgr_->Serialize(temp_table_ids); mem_mgr_->Serialize(temp_table_ids);
for (auto& id : temp_table_ids) { for (auto& id : temp_table_ids) {
compact_table_ids_.insert(id); sync_table_ids.insert(id);
} }
if (!temp_table_ids.empty()) { if (!temp_table_ids.empty()) {
@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() {
} }
// serialize memory data // serialize memory data
MemSerialize(); SyncMemData(compact_table_ids_);
// compactiong has been finished? // compactiong has been finished?
{ {

View File

@ -150,7 +150,7 @@ class DBImpl : public DB {
BackgroundBuildIndex(); BackgroundBuildIndex();
Status Status
MemSerialize(); SyncMemData(std::set<std::string>& sync_table_ids);
Status Status
GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types, GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types,

View File

@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
${CMAKE_COMMAND} -E echo ${CMAKE_COMMAND} -E echo
"Extracting ${package_file} to ${install_path}" "Extracting ${package_file} to ${install_path}"
COMMAND COMMAND
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path}
WORKING_DIRECTORY ${INDEX_BINARY_DIR} WORKING_DIRECTORY ${INDEX_BINARY_DIR}
) )

View File

@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object):
nlist = index_params["nlist"] nlist = index_params["nlist"]
logging.getLogger().info(index_params) logging.getLogger().info(index_params)
status, ids = connect.add_vectors(table, vectors) status, ids = connect.add_vectors(table, vectors)
# if not isinstance(index_type, int) or not isinstance(nlist, int): if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)):
with pytest.raises(Exception) as e: with pytest.raises(Exception) as e:
status = connect.create_index(table, index_params)
else:
status = connect.create_index(table, index_params) status = connect.create_index(table, index_params)
# else: assert not status.OK()
# status = connect.create_index(table, index_params)
# assert not status.OK()

View File

@ -54,12 +54,12 @@ def gen_long_str(num):
def gen_invalid_ips(): def gen_invalid_ips():
ips = [ ips = [
"255.0.0.0", # "255.0.0.0",
"255.255.0.0", # "255.255.0.0",
"255.255.255.0", # "255.255.255.0",
"255.255.255.255", # "255.255.255.255",
"127.0.0", "127.0.0",
"123.0.0.2", # "123.0.0.2",
"12-s", "12-s",
" ", " ",
"12 s", "12 s",
@ -114,12 +114,12 @@ def gen_invalid_uris():
# invalid ip # invalid ip
"tcp:// :%s" % port, "tcp:// :%s" % port,
"tcp://123.0.0.1:%s" % port, # "tcp://123.0.0.1:%s" % port,
"tcp://127.0.0:%s" % port, "tcp://127.0.0:%s" % port,
"tcp://255.0.0.0:%s" % port, "tcp://255.0.0.0:%s" % port,
"tcp://255.255.0.0:%s" % port, # "tcp://255.255.0.0:%s" % port,
"tcp://255.255.255.0:%s" % port, # "tcp://255.255.255.0:%s" % port,
"tcp://255.255.255.255:%s" % port, # "tcp://255.255.255.255:%s" % port,
"tcp://\n:%s" % port, "tcp://\n:%s" % port,
] ]
@ -263,9 +263,7 @@ def gen_invalid_index_types():
def gen_invalid_nlists(): def gen_invalid_nlists():
nlists = [ nlists = [
0,
-1, -1,
1000000000000001,
# None, # None,
[1,2,3], [1,2,3],
(1,2), (1,2),
@ -549,4 +547,4 @@ if __name__ == "__main__":
p.join() p.join()
time.sleep(3) time.sleep(3)
status, count = milvus.get_table_row_count(table) status, count = milvus.get_table_row_count(table)
assert count == process_num * loop_num assert count == process_num * loop_num