add s3 mock for unittest (#937)

* #815 change S3 client_ptr_ to shared_ptr

* #815 install test_storage to unittest

* #815 add MockS3Client.h for unittest

* #815 optimize MockS3Client.h

* #815 update unittest

* #815 update unittest

* #815 fix clang-format

* #815 use FIU for unittest

* #815 enable FIU in jenkins

* #815 update unittest

* #815 enable FIU in docker build

ci/jenkins/step/build.groovy

@@ -4,9 +4,9 @@ timeout(time: 75, unit: 'MINUTES') {
             def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
 
             if ("${BINARY_VERSION}" == "gpu") {
-                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c\""
+                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c -i\""
             } else {
-                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\""
+                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c -i\""
             }
             sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
         }

ci/scripts/build.sh

@@ -21,9 +21,10 @@ BUILD_COVERAGE="OFF"
 RUN_CPPLINT="OFF"
 GPU_VERSION="OFF"
 WITH_MKL="OFF"
+FIU_ENABLE="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "o:t:b:f:pgulcmh" arg
+while getopts "o:t:b:f:pgulcmih" arg
 do
         case $arg in
              o)
@@ -57,6 +58,9 @@ do
              m)
                 WITH_MKL="ON"
                 ;;
+             i)
+                FIU_ENABLE="ON"
+                ;;
              h) # help
                 echo "
 
@@ -71,10 +75,11 @@ parameter:
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
 -m: build with MKL(default: OFF)
+-i: build FIU_ENABLE(default: OFF)
 -h: help
 
 usage:
-./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-m] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-m] [-i] [-h]
                 "
                 exit 0
                 ;;
@@ -105,6 +110,7 @@ CMAKE_CMD="cmake \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \
 -DFAISS_SOURCE=AUTO \
+-DMILVUS_WITH_FIU=${FIU_ENABLE} \
 ${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

core/src/db/Utils.cpp

@@ -126,7 +126,7 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool
     if (minio_enable) {
         std::string table_path = options.path_ + TABLES_FOLDER + table_id;
 
-        auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
+        auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
         Status stat = storage_inst.DeleteObjects(table_path);
         if (!stat.ok()) {
             return stat;

core/src/storage/s3/S3ClientMock.h

@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <utility>
+
+#include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/utils/Outcome.h>
+#include <aws/core/utils/StringUtils.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/CreateBucketRequest.h>
+#include <aws/s3/model/DeleteBucketRequest.h>
+#include <aws/s3/model/DeleteObjectRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+
+namespace milvus {
+namespace storage {
+
+/*
+ * This is a class that represents a S3 Client which is used to mimic the put/get operations of a actual s3 client.
+ * During a put object, the body of the request is stored as well as the metadata of the request. This data is then
+ * populated into a get object result when a get operation is called.
+ */
+class S3ClientMock : public Aws::S3::S3Client {
+ public:
+    explicit S3ClientMock(Aws::Client::ClientConfiguration clientConfiguration = Aws::Client::ClientConfiguration())
+        : S3Client(Aws::Auth::AWSCredentials("", ""), clientConfiguration) {
+    }
+
+    Aws::S3::Model::CreateBucketOutcome
+    CreateBucket(const Aws::S3::Model::CreateBucketRequest& request) const override {
+        Aws::S3::Model::CreateBucketResult result;
+        return Aws::S3::Model::CreateBucketOutcome(std::move(result));
+    }
+
+    Aws::S3::Model::DeleteBucketOutcome
+    DeleteBucket(const Aws::S3::Model::DeleteBucketRequest& request) const override {
+        Aws::NoResult result;
+        return Aws::S3::Model::DeleteBucketOutcome(std::move(result));
+    }
+
+    Aws::S3::Model::PutObjectOutcome
+    PutObject(const Aws::S3::Model::PutObjectRequest& request) const override {
+        Aws::String key = request.GetKey();
+        std::shared_ptr<Aws::IOStream> body = request.GetBody();
+        aws_map_[key] = body;
+
+        Aws::S3::Model::PutObjectResult result;
+        return Aws::S3::Model::PutObjectOutcome(std::move(result));
+    }
+
+    Aws::S3::Model::GetObjectOutcome
+    GetObject(const Aws::S3::Model::GetObjectRequest& request) const override {
+        auto factory = request.GetResponseStreamFactory();
+        Aws::Utils::Stream::ResponseStream resp_stream(factory);
+
+        try {
+            std::shared_ptr<Aws::IOStream> body = aws_map_.at(request.GetKey());
+            Aws::String body_str((Aws::IStreamBufIterator(*body)), Aws::IStreamBufIterator());
+
+            resp_stream.GetUnderlyingStream().write(body_str.c_str(), body_str.length());
+            resp_stream.GetUnderlyingStream().flush();
+            Aws::AmazonWebServiceResult<Aws::Utils::Stream::ResponseStream> awsStream(
+                std::move(resp_stream), Aws::Http::HeaderValueCollection());
+
+            Aws::S3::Model::GetObjectResult result(std::move(awsStream));
+            return Aws::S3::Model::GetObjectOutcome(std::move(result));
+        } catch (...) {
+            return Aws::S3::Model::GetObjectOutcome();
+        }
+    }
+
+    Aws::S3::Model::ListObjectsOutcome
+    ListObjects(const Aws::S3::Model::ListObjectsRequest& request) const override {
+        /* TODO: add object key list into ListObjectsOutcome */
+
+        Aws::S3::Model::ListObjectsResult result;
+        return Aws::S3::Model::ListObjectsOutcome(std::move(result));
+    }
+
+    Aws::S3::Model::DeleteObjectOutcome
+    DeleteObject(const Aws::S3::Model::DeleteObjectRequest& request) const override {
+        Aws::String key = request.GetKey();
+        aws_map_.erase(key);
+        Aws::S3::Model::DeleteObjectResult result;
+        Aws::S3::Model::DeleteObjectOutcome(std::move(result));
+        return result;
+    }
+
+    mutable Aws::Map<Aws::String, std::shared_ptr<Aws::IOStream>> aws_map_;
+};
+
+}  // namespace storage
+}  // namespace milvus

core/src/storage/s3/S3ClientWrapper.cpp

@@ -22,12 +22,14 @@
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <fiu-local.h>
 #include <fstream>
 #include <iostream>
 #include <memory>
 #include <utility>
 
 #include "server/Config.h"
+#include "storage/s3/S3ClientMock.h"
 #include "storage/s3/S3ClientWrapper.h"
 #include "utils/Error.h"
 #include "utils/Log.h"
@@ -40,6 +42,7 @@ S3ClientWrapper::StartService() {
     server::Config& config = server::Config::GetInstance();
     bool minio_enable = false;
     CONFIG_CHECK(config.GetStorageConfigMinioEnable(minio_enable));
+    fiu_do_on("S3ClientWrapper.StartService.minio_disable", minio_enable = false);
     if (!minio_enable) {
         STORAGE_LOG_INFO << "MinIO not enabled!";
         return Status::OK();
@@ -52,29 +55,30 @@ S3ClientWrapper::StartService() {
     CONFIG_CHECK(config.GetStorageConfigMinioBucket(minio_bucket_));
 
     Aws::InitAPI(options_);
-    Aws::Client::ClientConfiguration cfg;
 
+    Aws::Client::ClientConfiguration cfg;
     cfg.endpointOverride = minio_address_ + ":" + minio_port_;
     cfg.scheme = Aws::Http::Scheme::HTTP;
     cfg.verifySSL = false;
-    client_ptr_ = new Aws::S3::S3Client(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg,
-                                        Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
-    if (client_ptr_ == nullptr) {
-        std::string str = "Cannot connect S3 server.";
-        return milvus::Status(SERVER_UNEXPECTED_ERROR, str);
+    client_ptr_ =
+        std::make_shared<Aws::S3::S3Client>(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg,
+                                            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
+
+    bool mock_enable = false;
+    fiu_do_on("S3ClientWrapper.StartService.mock_enable", mock_enable = true);
+    if (mock_enable) {
+        client_ptr_ = std::make_shared<S3ClientMock>();
     }
 
     return CreateBucket();
 }
 
-Status
+void
 S3ClientWrapper::StopService() {
     if (client_ptr_ != nullptr) {
-        delete client_ptr_;
         client_ptr_ = nullptr;
     }
     Aws::ShutdownAPI(options_);
-    return Status::OK();
 }
 
 Status
@@ -84,6 +88,7 @@ S3ClientWrapper::CreateBucket() {
 
     auto outcome = client_ptr_->CreateBucket(request);
 
+    fiu_do_on("S3ClientWrapper.CreateBucket.outcome.fail", outcome = Aws::S3::Model::CreateBucketOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         if (err.GetErrorType() != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) {
@@ -103,6 +108,7 @@ S3ClientWrapper::DeleteBucket() {
 
     auto outcome = client_ptr_->DeleteBucket(request);
 
+    fiu_do_on("S3ClientWrapper.DeleteBucket.outcome.fail", outcome = Aws::S3::Model::DeleteBucketOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: DeleteBucket: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -131,6 +137,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string
 
     auto outcome = client_ptr_->PutObject(request);
 
+    fiu_do_on("S3ClientWrapper.PutObjectFile.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -152,6 +159,7 @@ S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string&
 
     auto outcome = client_ptr_->PutObject(request);
 
+    fiu_do_on("S3ClientWrapper.PutObjectStr.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -169,6 +177,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string
 
     auto outcome = client_ptr_->GetObject(request);
 
+    fiu_do_on("S3ClientWrapper.GetObjectFile.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -191,6 +200,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte
 
     auto outcome = client_ptr_->GetObject(request);
 
+    fiu_do_on("S3ClientWrapper.GetObjectStr.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -217,6 +227,7 @@ S3ClientWrapper::ListObjects(std::vector<std::string>& object_list, const std::s
 
     auto outcome = client_ptr_->ListObjects(request);
 
+    fiu_do_on("S3ClientWrapper.ListObjects.outcome.fail", outcome = Aws::S3::Model::ListObjectsOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: ListObjects: " << err.GetExceptionName() << ": " << err.GetMessage();
@@ -244,6 +255,7 @@ S3ClientWrapper::DeleteObject(const std::string& object_name) {
 
     auto outcome = client_ptr_->DeleteObject(request);
 
+    fiu_do_on("S3ClientWrapper.DeleteObject.outcome.fail", outcome = Aws::S3::Model::DeleteObjectOutcome());
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
         STORAGE_LOG_ERROR << "ERROR: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage();

core/src/storage/s3/S3ClientWrapper.h

@@ -19,6 +19,7 @@
 
 #include <aws/core/Aws.h>
 #include <aws/s3/S3Client.h>
+#include <memory>
 #include <string>
 #include <vector>
 #include "storage/IStorage.h"
@@ -28,9 +29,6 @@ namespace storage {
 
 class S3ClientWrapper : public IStorage {
  public:
-    S3ClientWrapper() = default;
-    ~S3ClientWrapper() = default;
-
     static S3ClientWrapper&
     GetInstance() {
         static S3ClientWrapper wrapper;
@@ -39,7 +37,7 @@ class S3ClientWrapper : public IStorage {
 
     Status
     StartService();
-    Status
+    void
     StopService();
 
     Status
@@ -62,7 +60,7 @@ class S3ClientWrapper : public IStorage {
     DeleteObjects(const std::string& marker) override;
 
  private:
-    Aws::S3::S3Client* client_ptr_ = nullptr;
+    std::shared_ptr<Aws::S3::S3Client> client_ptr_;
     Aws::SDKOptions options_;
 
     std::string minio_address_;

core/unittest/storage/CMakeLists.txt

@@ -41,4 +41,4 @@ target_link_libraries(test_storage
         ${unittest_libs}
         )
 
-install(TARGETS test_storage DESTINATION bin)
+install(TARGETS test_storage DESTINATION unittest)

core/unittest/storage/test_s3_client.cpp

@@ -19,27 +19,31 @@
 #include <gtest/gtest.h>
 #include <fstream>
 #include <memory>
+#include <fiu-local.h>
+#include <fiu-control.h>
 
 #include "easyloggingpp/easylogging++.h"
 #include "server/Config.h"
-#include "storage/IStorage.h"
 #include "storage/s3/S3ClientWrapper.h"
+#include "storage/s3/S3IOReader.h"
+#include "storage/s3/S3IOWriter.h"
+#include "storage/IStorage.h"
 #include "storage/utils.h"
 
 INITIALIZE_EASYLOGGINGPP
 
 TEST_F(StorageTest, S3_CLIENT_TEST) {
+    fiu_init(0);
+
     const std::string filename = "/tmp/test_file_in";
+    const std::string filename_dummy = "/tmp/test_file_dummy";
     const std::string filename_out = "/tmp/test_file_out";
-    const std::string object_name = "/tmp/test_obj";
+    const std::string objname = "/tmp/test_obj";
+    const std::string objname_dummy = "/tmp/test_obj_dummy";
     const std::string content = "abcdefghijklmnopqrstuvwxyz";
 
-    std::string config_path(CONFIG_PATH);
-    config_path += CONFIG_FILE;
-    milvus::server::Config& config = milvus::server::Config::GetInstance();
-    ASSERT_TRUE(config.LoadConfigFile(config_path).ok());
-
-    auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
+    auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
+    fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0);
     ASSERT_TRUE(storage_inst.StartService().ok());
 
     ///////////////////////////////////////////////////////////////////////////
@@ -64,18 +68,137 @@ TEST_F(StorageTest, S3_CLIENT_TEST) {
     ///////////////////////////////////////////////////////////////////////////
     /* check PutObjectStr() and GetObjectStr() */
     {
-        ASSERT_TRUE(storage_inst.PutObjectStr(object_name, content).ok());
+        ASSERT_TRUE(storage_inst.PutObjectStr(objname, content).ok());
 
         std::string content_out;
-        ASSERT_TRUE(storage_inst.GetObjectStr(object_name, content_out).ok());
+        ASSERT_TRUE(storage_inst.GetObjectStr(objname, content_out).ok());
         ASSERT_TRUE(content_out == content);
     }
 
     ///////////////////////////////////////////////////////////////////////////
+    ASSERT_TRUE(storage_inst.DeleteObject(filename).ok());
+    ASSERT_TRUE(storage_inst.DeleteObject(objname).ok());
+
     ASSERT_TRUE(storage_inst.DeleteObjects("/tmp").ok());
 
     ASSERT_TRUE(storage_inst.DeleteBucket().ok());
 
-    ASSERT_TRUE(storage_inst.StopService().ok());
+    storage_inst.StopService();
 }
 
+TEST_F(StorageTest, S3_RW_TEST) {
+    fiu_init(0);
+
+    const std::string index_name = "/tmp/test_index";
+    const std::string content = "abcdefg";
+
+    auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
+    fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0);
+    ASSERT_TRUE(storage_inst.StartService().ok());
+
+    {
+        milvus::storage::S3IOWriter writer(index_name);
+        size_t len = content.length();
+        writer.write(&len, sizeof(len));
+        writer.write((void*)(content.data()), len);
+        ASSERT_TRUE(len + sizeof(len) == writer.length());
+    }
+
+    {
+        milvus::storage::S3IOReader reader(index_name);
+        size_t length = reader.length();
+        size_t rp = 0;
+        reader.seekg(rp);
+        std::string content_out;
+        while (rp < length) {
+            size_t len;
+            reader.read(&len, sizeof(len));
+            rp += sizeof(len);
+            reader.seekg(rp);
+
+            auto data = new char[len];
+            reader.read(data, len);
+            rp += len;
+            reader.seekg(rp);
+
+            content_out += std::string(data, len);
+
+            delete[] data;
+        }
+
+        ASSERT_TRUE(content == content_out);
+    }
+
+    storage_inst.StopService();
+}
+
+TEST_F(StorageTest, S3_FAIL_TEST) {
+    fiu_init(0);
+
+    const std::string filename = "/tmp/test_file_in";
+    const std::string filename_dummy = "/tmp/test_file_dummy";
+    const std::string filename_out = "/tmp/test_file_out";
+    const std::string objname = "/tmp/test_obj";
+    const std::string objname_dummy = "/tmp/test_obj_dummy";
+    const std::string content = "abcdefghijklmnopqrstuvwxyz";
+
+    auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
+
+    fiu_enable("S3ClientWrapper.StartService.minio_disable", 1, NULL, 0);
+    ASSERT_TRUE(storage_inst.StartService().ok());
+    fiu_disable("S3ClientWrapper.StartService.minio_disable");
+
+    fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0);
+    ASSERT_TRUE(storage_inst.StartService().ok());
+    fiu_disable("S3ClientWrapper.StartService.mock_enable");
+
+    fiu_enable("S3ClientWrapper.CreateBucket.outcome.fail", 1, NULL, 0);
+    ASSERT_FALSE(storage_inst.CreateBucket().ok());
+    fiu_disable("S3ClientWrapper.CreateBucket.outcome.fail");
+
+    ///////////////////////////////////////////////////////////////////////////
+    /* check PutObjectFile() and GetObjectFile() */
+    {
+        fiu_enable("S3ClientWrapper.PutObjectFile.outcome.fail", 1, NULL, 0);
+        ASSERT_FALSE(storage_inst.PutObjectFile(filename, filename).ok());
+        fiu_disable("S3ClientWrapper.PutObjectFile.outcome.fail");
+
+        fiu_enable("S3ClientWrapper.GetObjectFile.outcome.fail", 1, NULL, 0);
+        ASSERT_FALSE(storage_inst.GetObjectFile(filename, filename_out).ok());
+        fiu_disable("S3ClientWrapper.GetObjectFile.outcome.fail");
+
+        ASSERT_FALSE(storage_inst.PutObjectFile(filename_dummy, filename_dummy).ok());
+        ASSERT_FALSE(storage_inst.GetObjectFile(filename_dummy, filename_out).ok());
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+    /* check PutObjectStr() and GetObjectStr() */
+    {
+        fiu_enable("S3ClientWrapper.PutObjectStr.outcome.fail", 1, NULL, 0);
+        ASSERT_FALSE(storage_inst.PutObjectStr(objname, content).ok());
+        fiu_disable("S3ClientWrapper.PutObjectStr.outcome.fail");
+
+        std::string content_out;
+        fiu_enable("S3ClientWrapper.GetObjectStr.outcome.fail", 1, NULL, 0);
+        ASSERT_FALSE(storage_inst.GetObjectStr(objname, content_out).ok());
+        fiu_disable("S3ClientWrapper.GetObjectStr.outcome.fail");
+
+        ASSERT_FALSE(storage_inst.GetObjectStr(objname_dummy, content_out).ok());
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+    fiu_enable("S3ClientWrapper.DeleteObject.outcome.fail", 1, NULL, 0);
+    ASSERT_FALSE(storage_inst.DeleteObject(filename).ok());
+    fiu_disable("S3ClientWrapper.DeleteObject.outcome.fail");
+
+    fiu_enable("S3ClientWrapper.ListObjects.outcome.fail", 1, NULL, 0);
+    ASSERT_FALSE(storage_inst.DeleteObjects("/tmp").ok());
+    fiu_disable("S3ClientWrapper.ListObjects.outcome.fail");
+    ASSERT_TRUE(storage_inst.DeleteObjects("/tmp").ok());
+
+    fiu_enable("S3ClientWrapper.DeleteBucket.outcome.fail", 1, NULL, 0);
+    ASSERT_FALSE(storage_inst.DeleteBucket().ok());
+    fiu_disable("S3ClientWrapper.DeleteBucket.outcome.fail");
+
+    storage_inst.StopService();
+}

core/unittest/storage/utils.cpp

@@ -18,6 +18,7 @@
 #include <fstream>
 #include <string>
 
+#include "server/Config.h"
 #include "storage/utils.h"
 #include "utils/CommonUtil.h"
 
@@ -50,7 +51,11 @@ void
 StorageTest::SetUp() {
     std::string config_path(CONFIG_PATH);
     milvus::server::CommonUtil::CreateDirectory(config_path);
-    WriteToFile(config_path + CONFIG_FILE, CONFIG_STR);
+    config_path += CONFIG_FILE;
+    WriteToFile(config_path, CONFIG_STR);
+
+    milvus::server::Config& config = milvus::server::Config::GetInstance();
+    ASSERT_TRUE(config.LoadConfigFile(config_path).ok());
 }
 
 void

docker-compose.yml

@@ -40,7 +40,7 @@ services:
       - milvus
     command: &ubuntu-command >
       /bin/bash -c "
-        /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c
+        /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i
         /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql"
 
   centos-core:
@@ -60,7 +60,7 @@ services:
       - milvus
     command: &centos-command >
       /bin/bash --login -c "
-        /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c
+        /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i
         /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql"
 
 networks: