fix random crash of INSERT_DUPLICATE_ID case (#2557)

* fix random crash of INSERT_DUPLICATE_ID case Signed-off-by: yhmo <yihua.mo@zilliz.com> * cahnge ver Signed-off-by: yhmo <yihua.mo@zilliz.com> * sdk readme Signed-off-by: yhmo <yihua.mo@zilliz.com> * fix ut failure Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com>
2020-06-17 11:50:19 +08:00 · 2020-06-17 11:50:19 +08:00 · 36d2214fc9
parent bd7e0ee5c9
commit 36d2214fc9
4 changed files with 54 additions and 34 deletions
--- a/core/unittest/db/test_db.cpp
+++ b/core/unittest/db/test_db.cpp
@ -250,13 +250,13 @@ TEST_F(DBTest, DB_TEST) {
    {
        auto options = GetOptions();
        options.meta_.backend_uri_ = "dummy";
-        ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
+        ASSERT_ANY_THROW(BuildDB(options));

        options.meta_.backend_uri_ = "mysql://root:123456@127.0.0.1:3306/test";
-        ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
+        ASSERT_ANY_THROW(BuildDB(options));

        options.meta_.backend_uri_ = "dummy://root:123456@127.0.0.1:3306/test";
-        ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
+        ASSERT_ANY_THROW(BuildDB(options));
    }
 }

@ -1116,17 +1116,13 @@ TEST_F(DBTestWALRecovery, RECOVERY_WITH_NO_ERROR) {
    milvus::engine::ResultDistances result_distances;
    milvus::engine::VectorsData qxb;
    BuildVectors(qb, 0, qxb);
-    stat = db_->Query(dummy_context_,
-            collection_info.collection_id_, {}, topk, json_params, qxb, result_ids, result_distances);
-    ASSERT_TRUE(stat.ok());
-    ASSERT_NE(result_ids.size() / topk, qb);

    fiu_init(0);
    fiu_enable("DBImpl.ExexWalRecord.return", 1, nullptr, 0);
-    db_ = nullptr;
+    db_ = nullptr; // don't use FreeDB(), this case needs keep the meta
    fiu_disable("DBImpl.ExexWalRecord.return");
    auto options = GetOptions();
-    db_ = milvus::engine::DBFactory::Build(options);
+    BuildDB(options);

    result_ids.clear();
    result_distances.clear();
@ -1158,13 +1154,13 @@ TEST_F(DBTestWALRecovery_Error, RECOVERY_WITH_INVALID_LOG_FILE) {

    fiu_init(0);
    fiu_enable("DBImpl.ExexWalRecord.return", 1, nullptr, 0);
-    db_ = nullptr;
+    FreeDB();
    fiu_disable("DBImpl.ExexWalRecord.return");

    auto options = GetOptions();
    // delete wal log file so that recovery will failed when start db next time.
    boost::filesystem::remove(options.mxlog_path_ + "0.wal");
-    ASSERT_ANY_THROW(db_ = milvus::engine::DBFactory::Build(options));
+    ASSERT_ANY_THROW(BuildDB(options));
 }

 TEST_F(DBTest2, FLUSH_NON_EXISTING_COLLECTION) {
@ -1256,7 +1252,6 @@ TEST_F(DBTest2, GET_VECTOR_BY_ID_INVALID_TEST) {
    fiu_disable("bloom_filter_nullptr");
 }

-
 TEST_F(DBTest2, GET_VECTOR_IDS_TEST) {
    milvus::engine::meta::CollectionSchema collection_schema = BuildCollectionSchema();
    auto stat = db_->CreateCollection(collection_schema);
@ -1316,11 +1311,11 @@ TEST_F(DBTest2, GET_VECTOR_IDS_TEST) {
 TEST_F(DBTest2, INSERT_DUPLICATE_ID) {
    auto options = GetOptions();
    options.wal_enable_ = false;
-    db_ = milvus::engine::DBFactory::Build(options);
+    BuildDB(options);

    milvus::engine::meta::CollectionSchema collection_schema = BuildCollectionSchema();
    auto stat = db_->CreateCollection(collection_schema);
-    ASSERT_TRUE(stat.ok()) << " CreateCollection: " << stat.message();
+    ASSERT_TRUE(stat.ok());

    uint64_t size = 20;
    milvus::engine::VectorsData vector;
@ -1331,10 +1326,10 @@ TEST_F(DBTest2, INSERT_DUPLICATE_ID) {
    }

    stat = db_->InsertVectors(COLLECTION_NAME, "", vector);
-    ASSERT_TRUE(stat.ok()) << " InsertVectors: " << stat.message();
+    ASSERT_TRUE(stat.ok());

    stat = db_->Flush(COLLECTION_NAME);
-    ASSERT_TRUE(stat.ok()) << " Flush: " << stat.message();
+    ASSERT_TRUE(stat.ok());
 }

 /*
--- a/core/unittest/db/utils.cpp
+++ b/core/unittest/db/utils.cpp
@ -26,7 +26,6 @@
 #include "db/DBFactory.h"
 #include "db/Options.h"

-
 #ifdef MILVUS_GPU_VERSION
 #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 #endif
@ -255,7 +254,7 @@ DBTest::SetUp() {

    auto options = GetOptions();
    options.insert_cache_immediately_ = true;
-    db_ = milvus::engine::DBFactory::Build(options);
+    BuildDB(options);

    std::string config_path(options.meta_.path_ + CONFIG_FILE);
    WriteToFile(config_path, CONFIG_STR);
@ -263,10 +262,7 @@ DBTest::SetUp() {

 void
 DBTest::TearDown() {
-    if (db_) {
-        db_->Stop();
-        db_->DropAll();
-    }
+    FreeDB();

    milvus::scheduler::JobMgrInst::GetInstance()->Stop();
    milvus::scheduler::SchedInst::GetInstance()->Stop();
@ -280,6 +276,21 @@ DBTest::TearDown() {
    boost::filesystem::remove_all(options.meta_.path_);
 }

+void
+DBTest::BuildDB(const milvus::engine::DBOptions& options) {
+    FreeDB();
+    db_ = milvus::engine::DBFactory::Build(options);
+}
+
+void
+DBTest::FreeDB() {
+    if (db_) {
+        db_->Stop();
+        db_->DropAll();
+        db_ = nullptr;
+    }
+}
+
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 milvus::engine::DBOptions
 DBTest2::GetOptions() {
--- a/core/unittest/db/utils.h
+++ b/core/unittest/db/utils.h
@ -72,6 +72,12 @@ class DBTest : public BaseTest {
    SetUp() override;
    void
    TearDown() override;
+
+    void
+    BuildDB(const milvus::engine::DBOptions& options);
+
+    void
+    FreeDB();
 };

 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/sdk/README.md
+++ b/sdk/README.md
@ -32,7 +32,7 @@ Run C++ example:

 ### Create your own C++ client project

-Create a folder for the project, and copy C++ SDK header and library files into it.
+- Create a folder for the project, and copy C++ SDK header and library files into it.

 ```shell
 # create project folder
@ -41,15 +41,14 @@ Create a folder for the project, and copy C++ SDK header and library files into
 
 # copy necessary files
 $ cp [Milvus root path]/sdk/cmake_build/libmilvus_sdk.so .
- $ cp [Milvus root path]/sdk/include/MilvusApi.h .
- $ cp [Milvus root path]/sdk/include/Status.h .
+ $ cp -r [Milvus root path]/sdk/include .
 ```

-Create file `main.cpp` in the project folder, and copy the following code into it:
+- Create file `main.cpp` in the project folder, and copy the following code into it:

 ```c++
-#include "./MilvusApi.h"
-#include "./Status.h"
+#include "./include/MilvusApi.h"
+#include "./include/Status.h"

 int main() {
  // connect to milvus server
@ -64,7 +63,7 @@ int main() {
 }
 ```

-Create file `CMakeLists.txt` in the project folder, and copy the following code into it:
+- Create file `CMakeLists.txt` in the project folder, and copy the following code into it:

 ```bash
 cmake_minimum_required(VERSION 3.14)
@ -77,18 +76,20 @@ Create file `CMakeLists.txt` in the project folder, and copy the following code
         pthread)
 ```

-Now there are 5 files in your project:
+- Now the file structure of your project:

 ```shell
 MyMilvusClient
 |-CMakeLists.txt
 |-main.cpp
 |-libmilvus_sdk.so
- |-MilvusApi.h
- |-Status.h
+ |-include
+     |-MilvusApi.h
+     |-Status.h
+     |-......
 ```

-Build the project:
+- Build the project:

 ```shell
 $ mkdir cmake_build
@ -97,8 +98,15 @@ Build the project:
 $ make
 ```

-Run your client program:
+- Run your client program:

 ```shell
 $ ./milvus_client
 ```
+
+### Troubleshooting
+
+- compile error "cannot find -lz"
+```shell
+ $ apt-get install zlib1g-dev.
+```