From d2c2f99f7233cfe00b6147aac99772a7eb29a9d2 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 11:20:24 +0800 Subject: [PATCH 1/7] add badge in README.md --- README.md | 2 ++ README_CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 4a1151dbf8..3cb1b34c25 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ ![Release](https://img.shields.io/badge/release-v0.5.1-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + [中文版](README_CN.md) ## What is Milvus diff --git a/README_CN.md b/README_CN.md index 8f4d4b05aa..5daef16d24 100644 --- a/README_CN.md +++ b/README_CN.md @@ -7,6 +7,8 @@ ![Release](https://img.shields.io/badge/release-v0.5.0-orange) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + - [Slack 频道](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) - [Twitter](https://twitter.com/milvusio) - [Facebook](https://www.facebook.com/io.milvus.5) From a3a8e222aa10f80018960c9311070330ccd4aad7 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 25 Nov 2019 14:53:23 +0800 Subject: [PATCH 2/7] remove invalid params --- tests/milvus_python_test/test_index.py | 10 +++++----- tests/milvus_python_test/utils.py | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 39aadb9d33..b253cf02a3 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object): nlist = index_params["nlist"] logging.getLogger().info(index_params) status, ids = connect.add_vectors(table, vectors) - # if not isinstance(index_type, int) or not isinstance(nlist, int): - with pytest.raises(Exception) as e: + if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)): + with pytest.raises(Exception) as e: + status = connect.create_index(table, index_params) + else: status = connect.create_index(table, index_params) - # else: - # status = connect.create_index(table, index_params) - # assert not status.OK() + assert not status.OK() diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py index 159c8407c6..1686ad7129 100644 --- a/tests/milvus_python_test/utils.py +++ b/tests/milvus_python_test/utils.py @@ -54,12 +54,12 @@ def gen_long_str(num): def gen_invalid_ips(): ips = [ - "255.0.0.0", - "255.255.0.0", - "255.255.255.0", - "255.255.255.255", + # "255.0.0.0", + # "255.255.0.0", + # "255.255.255.0", + # "255.255.255.255", "127.0.0", - "123.0.0.2", + # "123.0.0.2", "12-s", " ", "12 s", @@ -114,12 +114,12 @@ def gen_invalid_uris(): # invalid ip "tcp:// :%s" % port, - "tcp://123.0.0.1:%s" % port, + # "tcp://123.0.0.1:%s" % port, "tcp://127.0.0:%s" % port, "tcp://255.0.0.0:%s" % port, - "tcp://255.255.0.0:%s" % port, - "tcp://255.255.255.0:%s" % port, - "tcp://255.255.255.255:%s" % port, + # "tcp://255.255.0.0:%s" % port, + # "tcp://255.255.255.0:%s" % port, + # "tcp://255.255.255.255:%s" % port, "tcp://\n:%s" % port, ] @@ -263,9 +263,7 @@ def gen_invalid_index_types(): def gen_invalid_nlists(): nlists = [ - 0, -1, - 1000000000000001, # None, [1,2,3], (1,2), @@ -549,4 +547,4 @@ if __name__ == "__main__": p.join() time.sleep(3) status, count = milvus.get_table_row_count(table) - assert count == process_num * loop_num \ No newline at end of file + assert count == process_num * loop_num From 78252ddaca6ac190fa7ca0ccdeec17135452bbe4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 14:53:42 +0800 Subject: [PATCH 3/7] disable untar verbose --- core/src/index/cmake/BuildUtilsCore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/cmake/BuildUtilsCore.cmake b/core/src/index/cmake/BuildUtilsCore.cmake index 1bf6ba2af4..2261ce3e29 100644 --- a/core/src/index/cmake/BuildUtilsCore.cmake +++ b/core/src/index/cmake/BuildUtilsCore.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${INDEX_BINARY_DIR} ) From f97e62de1e33fc5c55e7529442b9bde89a64a2eb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 15:31:51 +0800 Subject: [PATCH 4/7] set CMAKE_BUILD_TYPE default values is Release --- core/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 671330c192..b24ac8c00b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -31,6 +31,10 @@ GET_CURRENT_TIME(BUILD_TIME) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") +if (NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") +endif () + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) From d3812c2e5e3838cd8f8034a2e726185bd04315d6 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 16:03:51 +0800 Subject: [PATCH 5/7] update CMakeLists.txt --- core/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b24ac8c00b..ba391c2d6b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -35,9 +35,17 @@ if (NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") endif () +set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]") + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -47,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "") endif () set(MILVUS_VERSION "${GIT_BRANCH_NAME}") -string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}") if (CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") From 0c7c4586c059a5fcb4f46bf0003a6bfce596d043 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 25 Nov 2019 16:41:48 +0800 Subject: [PATCH 6/7] #513 DELETE_BY_RANGE sometimes failed --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 14 +++++++++----- core/src/db/DBImpl.h | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e427dcb9..84bd8aed51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building +- \#513 - DELETE_BY_RANGE sometimes failed ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 51ea665064..1c18c22409 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -105,7 +105,8 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); // makesure all memory data serialized - MemSerialize(); + std::set sync_table_ids; + SyncMemData(sync_table_ids); // wait compaction/buildindex finish bg_timer_thread_.join(); @@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { return SHUTDOWN_ERROR; } - Status status; + // serialize memory data + std::set sync_table_ids; + auto status = SyncMemData(sync_table_ids); + { std::unique_lock lock(build_index_mutex_); @@ -588,12 +592,12 @@ DBImpl::StartMetricTask() { } Status -DBImpl::MemSerialize() { +DBImpl::SyncMemData(std::set& sync_table_ids) { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); for (auto& id : temp_table_ids) { - compact_table_ids_.insert(id); + sync_table_ids.insert(id); } if (!temp_table_ids.empty()) { @@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() { } // serialize memory data - MemSerialize(); + SyncMemData(compact_table_ids_); // compactiong has been finished? { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index bff56efded..82a5d3096b 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -150,7 +150,7 @@ class DBImpl : public DB { BackgroundBuildIndex(); Status - MemSerialize(); + SyncMemData(std::set& sync_table_ids); Status GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, From c2971cfd1e97880621570290c0fe005b3761f2d2 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 25 Nov 2019 16:56:41 +0800 Subject: [PATCH 7/7] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84bd8aed51..060aaed1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building -- \#513 - DELETE_BY_RANGE sometimes failed +- \#513 - Unittest DELETE_BY_RANGE sometimes failed ## Feature - \#12 - Pure CPU version for Milvus