mirror of https://github.com/milvus-io/milvus.git
Merge branch '0.6.0' into issue-509
commit
038850d6a9
|
@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
|
||||||
- \#458 - Index data is not compatible between 0.5 and 0.6
|
- \#458 - Index data is not compatible between 0.5 and 0.6
|
||||||
- \#486 - gpu no usage during index building
|
- \#486 - gpu no usage during index building
|
||||||
- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params
|
- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params
|
||||||
|
- \#513 - Unittest DELETE_BY_RANGE sometimes failed
|
||||||
|
|
||||||
## Feature
|
## Feature
|
||||||
- \#12 - Pure CPU version for Milvus
|
- \#12 - Pure CPU version for Milvus
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
|
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
|
||||||
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
|
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
|
||||||
![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen)
|
![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen)
|
||||||
|
[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
|
||||||
|
|
||||||
[中文版](README_CN.md) | [日本語版](README_JP.md)
|
[中文版](README_CN.md) | [日本語版](README_JP.md)
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,8 @@
|
||||||
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
|
[![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
|
||||||
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
|
![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
|
||||||
![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen)
|
![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen)
|
||||||
|
[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
|
||||||
|
|
||||||
|
|
||||||
# 欢迎来到 Milvus
|
# 欢迎来到 Milvus
|
||||||
|
|
||||||
|
|
|
@ -31,9 +31,21 @@ GET_CURRENT_TIME(BUILD_TIME)
|
||||||
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
|
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
|
||||||
message(STATUS "Build time = ${BUILD_TIME}")
|
message(STATUS "Build time = ${BUILD_TIME}")
|
||||||
|
|
||||||
|
if (NOT DEFINED CMAKE_BUILD_TYPE)
|
||||||
|
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.")
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]")
|
||||||
|
|
||||||
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
|
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
|
||||||
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
|
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
|
||||||
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
||||||
|
if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
|
||||||
|
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
||||||
|
endif ()
|
||||||
|
if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
|
||||||
|
execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
|
||||||
|
endif ()
|
||||||
ENDMACRO(GET_GIT_BRANCH_NAME)
|
ENDMACRO(GET_GIT_BRANCH_NAME)
|
||||||
|
|
||||||
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
|
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
|
||||||
|
@ -43,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "")
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
|
set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
|
||||||
string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
|
string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}")
|
||||||
|
|
||||||
if (CMAKE_BUILD_TYPE STREQUAL "Release")
|
if (CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||||
set(BUILD_TYPE "Release")
|
set(BUILD_TYPE "Release")
|
||||||
|
|
|
@ -105,7 +105,8 @@ DBImpl::Stop() {
|
||||||
shutting_down_.store(true, std::memory_order_release);
|
shutting_down_.store(true, std::memory_order_release);
|
||||||
|
|
||||||
// makesure all memory data serialized
|
// makesure all memory data serialized
|
||||||
MemSerialize();
|
std::set<std::string> sync_table_ids;
|
||||||
|
SyncMemData(sync_table_ids);
|
||||||
|
|
||||||
// wait compaction/buildindex finish
|
// wait compaction/buildindex finish
|
||||||
bg_timer_thread_.join();
|
bg_timer_thread_.join();
|
||||||
|
@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
|
||||||
return SHUTDOWN_ERROR;
|
return SHUTDOWN_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
Status status;
|
// serialize memory data
|
||||||
|
std::set<std::string> sync_table_ids;
|
||||||
|
auto status = SyncMemData(sync_table_ids);
|
||||||
|
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(build_index_mutex_);
|
std::unique_lock<std::mutex> lock(build_index_mutex_);
|
||||||
|
|
||||||
|
@ -588,12 +592,12 @@ DBImpl::StartMetricTask() {
|
||||||
}
|
}
|
||||||
|
|
||||||
Status
|
Status
|
||||||
DBImpl::MemSerialize() {
|
DBImpl::SyncMemData(std::set<std::string>& sync_table_ids) {
|
||||||
std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
|
std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
|
||||||
std::set<std::string> temp_table_ids;
|
std::set<std::string> temp_table_ids;
|
||||||
mem_mgr_->Serialize(temp_table_ids);
|
mem_mgr_->Serialize(temp_table_ids);
|
||||||
for (auto& id : temp_table_ids) {
|
for (auto& id : temp_table_ids) {
|
||||||
compact_table_ids_.insert(id);
|
sync_table_ids.insert(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!temp_table_ids.empty()) {
|
if (!temp_table_ids.empty()) {
|
||||||
|
@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// serialize memory data
|
// serialize memory data
|
||||||
MemSerialize();
|
SyncMemData(compact_table_ids_);
|
||||||
|
|
||||||
// compactiong has been finished?
|
// compactiong has been finished?
|
||||||
{
|
{
|
||||||
|
|
|
@ -150,7 +150,7 @@ class DBImpl : public DB {
|
||||||
BackgroundBuildIndex();
|
BackgroundBuildIndex();
|
||||||
|
|
||||||
Status
|
Status
|
||||||
MemSerialize();
|
SyncMemData(std::set<std::string>& sync_table_ids);
|
||||||
|
|
||||||
Status
|
Status
|
||||||
GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types,
|
GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types,
|
||||||
|
|
|
@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
|
||||||
${CMAKE_COMMAND} -E echo
|
${CMAKE_COMMAND} -E echo
|
||||||
"Extracting ${package_file} to ${install_path}"
|
"Extracting ${package_file} to ${install_path}"
|
||||||
COMMAND
|
COMMAND
|
||||||
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
|
${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path}
|
||||||
WORKING_DIRECTORY ${INDEX_BINARY_DIR}
|
WORKING_DIRECTORY ${INDEX_BINARY_DIR}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object):
|
||||||
nlist = index_params["nlist"]
|
nlist = index_params["nlist"]
|
||||||
logging.getLogger().info(index_params)
|
logging.getLogger().info(index_params)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(table, vectors)
|
||||||
# if not isinstance(index_type, int) or not isinstance(nlist, int):
|
if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)):
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
|
status = connect.create_index(table, index_params)
|
||||||
|
else:
|
||||||
status = connect.create_index(table, index_params)
|
status = connect.create_index(table, index_params)
|
||||||
# else:
|
assert not status.OK()
|
||||||
# status = connect.create_index(table, index_params)
|
|
||||||
# assert not status.OK()
|
|
||||||
|
|
|
@ -54,12 +54,12 @@ def gen_long_str(num):
|
||||||
|
|
||||||
def gen_invalid_ips():
|
def gen_invalid_ips():
|
||||||
ips = [
|
ips = [
|
||||||
"255.0.0.0",
|
# "255.0.0.0",
|
||||||
"255.255.0.0",
|
# "255.255.0.0",
|
||||||
"255.255.255.0",
|
# "255.255.255.0",
|
||||||
"255.255.255.255",
|
# "255.255.255.255",
|
||||||
"127.0.0",
|
"127.0.0",
|
||||||
"123.0.0.2",
|
# "123.0.0.2",
|
||||||
"12-s",
|
"12-s",
|
||||||
" ",
|
" ",
|
||||||
"12 s",
|
"12 s",
|
||||||
|
@ -114,12 +114,12 @@ def gen_invalid_uris():
|
||||||
|
|
||||||
# invalid ip
|
# invalid ip
|
||||||
"tcp:// :%s" % port,
|
"tcp:// :%s" % port,
|
||||||
"tcp://123.0.0.1:%s" % port,
|
# "tcp://123.0.0.1:%s" % port,
|
||||||
"tcp://127.0.0:%s" % port,
|
"tcp://127.0.0:%s" % port,
|
||||||
"tcp://255.0.0.0:%s" % port,
|
"tcp://255.0.0.0:%s" % port,
|
||||||
"tcp://255.255.0.0:%s" % port,
|
# "tcp://255.255.0.0:%s" % port,
|
||||||
"tcp://255.255.255.0:%s" % port,
|
# "tcp://255.255.255.0:%s" % port,
|
||||||
"tcp://255.255.255.255:%s" % port,
|
# "tcp://255.255.255.255:%s" % port,
|
||||||
"tcp://\n:%s" % port,
|
"tcp://\n:%s" % port,
|
||||||
|
|
||||||
]
|
]
|
||||||
|
@ -263,9 +263,7 @@ def gen_invalid_index_types():
|
||||||
|
|
||||||
def gen_invalid_nlists():
|
def gen_invalid_nlists():
|
||||||
nlists = [
|
nlists = [
|
||||||
0,
|
|
||||||
-1,
|
-1,
|
||||||
1000000000000001,
|
|
||||||
# None,
|
# None,
|
||||||
[1,2,3],
|
[1,2,3],
|
||||||
(1,2),
|
(1,2),
|
||||||
|
@ -549,4 +547,4 @@ if __name__ == "__main__":
|
||||||
p.join()
|
p.join()
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
status, count = milvus.get_table_row_count(table)
|
status, count = milvus.get_table_row_count(table)
|
||||||
assert count == process_num * loop_num
|
assert count == process_num * loop_num
|
||||||
|
|
Loading…
Reference in New Issue