From 5950deddfc0a86d12ad766a855a557044600dd22 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Fri, 20 Mar 2020 10:24:27 +0800 Subject: [PATCH] #1678 remove CUSTOMIZATION macro (#1707) * #1678 remove CUSTOMIZATION macro Signed-off-by: yudong.cai * disable faiss_benchmark Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/CMakeLists.txt | 4 -- core/cmake/DefineOptions.cmake | 2 - core/src/db/engine/ExecutionEngineImpl.cpp | 2 - core/src/index/CMakeLists.txt | 7 --- core/src/index/cmake/DefineOptionsCore.cmake | 2 - .../knowhere/index/vector_index/IndexIVF.cpp | 2 +- .../vector_index/gpu/IndexIVFSQHybrid.cpp | 2 +- .../index/vector_index/gpu/IndexIVFSQHybrid.h | 3 +- .../index/vector_index/helpers/Cloner.cpp | 4 +- .../faiss_benchmark/faiss_benchmark_test.cpp | 44 +++---------------- .../faiss_benchmark/faiss_bitset_test.cpp | 23 +++------- .../index/unittest/test_customized_index.cpp | 2 +- core/src/index/unittest/test_ivf.cpp | 9 ---- core/src/index/unittest/test_vecindex.cpp | 6 +-- core/src/scheduler/SchedInst.h | 2 - .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 5 +-- core/src/utils/ValidationUtil.cpp | 6 +-- core/unittest/db/test_db.cpp | 6 --- core/unittest/server/test_util.cpp | 2 +- 20 files changed, 27 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c0820834d..efb767ba85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Please mark all change in change log and use the issue from GitHub - \#1619 Improve compact performance - \#1649 Fix Milvus crash on old CPU - \#1653 IndexFlat (SSE) and IndexBinaryFlat performance improvement for small NQ +- \#1678 Remove CUSTOMIZATION macro ## Task diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 3b54df420d..34de23399a 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -153,10 +153,6 @@ if (MILVUS_USE_CCACHE) endif (CCACHE_FOUND) endif () -if (CUSTOMIZATION) - add_compile_definitions(CUSTOMIZATION) -endif () - if (MILVUS_GPU_VERSION) message(STATUS "Building Milvus GPU version") add_compile_definitions("MILVUS_GPU_VERSION") diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index f5e013b2be..d5eab6a6a0 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -45,8 +45,6 @@ set_option_category("Milvus Build Option") define_option(MILVUS_GPU_VERSION "Build GPU version" OFF) -define_option(CUSTOMIZATION "Build with customized FAISS library" ON) - #---------------------------------------------------------------------- set_option_category("Thirdparty") diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index bd8e6dd215..6741cf610f 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -186,13 +186,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, mode); break; } -#ifdef CUSTOMIZATION #ifdef MILVUS_GPU_VERSION case EngineType::FAISS_IVFSQ8H: { index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, mode); break; } -#endif #endif case EngineType::FAISS_BIN_IDMAP: { index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, mode); diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index fad8b5c981..232fa80181 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -66,11 +66,6 @@ include(ExternalProject) include(DefineOptionsCore) include(BuildUtilsCore) -if (CUSTOMIZATION) - add_compile_definitions(CUSTOMIZATION) -endif () - -set(KNOWHERE_CPU_VERSION false) if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION) message(STATUS "Building Knowhere GPU version") add_compile_definitions("MILVUS_GPU_VERSION") @@ -79,8 +74,6 @@ if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") else () message(STATUS "Building Knowhere CPU version") - set(KNOWHERE_CPU_VERSION true) - add_compile_definitions("MILVUS_CPU_VERSION") endif () include(ThirdPartyPackagesCore) diff --git a/core/src/index/cmake/DefineOptionsCore.cmake b/core/src/index/cmake/DefineOptionsCore.cmake index 3ada0e2a35..7ef155d416 100644 --- a/core/src/index/cmake/DefineOptionsCore.cmake +++ b/core/src/index/cmake/DefineOptionsCore.cmake @@ -49,8 +49,6 @@ else () define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF) endif () -define_option(CUSTOMIZATION "Build with customized FAISS library" ON) - #---------------------------------------------------------------------- set_option_category("Thirdparty") diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 46db3bc2fb..185948e63a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -301,7 +301,7 @@ IVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_ void IVF::SealImpl() { -#ifdef CUSTOMIZATION +#ifdef MILVUS_GPU_VERSION faiss::Index* index = index_.get(); auto idx = dynamic_cast(index); if (idx != nullptr) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp index 9cb24f0ffd..edd6db1c6f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp @@ -26,7 +26,7 @@ namespace milvus { namespace knowhere { -#ifdef CUSTOMIZATION +#ifdef MILVUS_GPU_VERSION void IVFSQHybrid::Train(const DatasetPtr& dataset_ptr, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.h b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.h index ea4a317c79..578ecd0d9f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.h @@ -23,7 +23,8 @@ namespace milvus { namespace knowhere { -#ifdef CUSTOMIZATION +#ifdef MILVUS_GPU_VERSION + struct FaissIVFQuantizer : public Quantizer { faiss::gpu::GpuIndexFlat* quantizer = nullptr; int64_t gpu_id; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index b2aca67128..d6e7532d33 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -9,6 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License +#ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexIDMAP.h" @@ -39,13 +40,11 @@ VecIndexPtr CopyCpuToGpu(const VecIndexPtr& index, const int64_t device_id, const Config& config) { VecIndexPtr result; auto uids = index->GetUids(); -#ifdef CUSTOMIZATION if (auto device_index = std::dynamic_pointer_cast(index)) { result = device_index->CopyCpuToGpu(device_id, config); result->SetUids(uids); return result; } -#endif if (auto device_index = std::dynamic_pointer_cast(index)) { result = device_index->CopyGpuToGpu(device_id, config); @@ -72,3 +71,4 @@ CopyCpuToGpu(const VecIndexPtr& index, const int64_t device_id, const Config& co } // namespace cloner } // namespace knowhere } // namespace milvus +#endif diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 95ba331008..ea01c61b35 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -9,10 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#define USE_FAISS_V_0_3_0 - #include - #include #include #include @@ -25,31 +22,14 @@ #include #include #include -#include -#include -#include - -#ifdef USE_FAISS_V_0_3_0 // faiss_0.3.0 - #include -#include -#include - -#else // faiss_0.2.1 - -#include -#include -#include -#include -#include - -#endif - -#ifdef CUSTOMIZATION -#include -#else +#include #include -#endif +#include +#include +#include +#include +#include /***************************************************** * To run this test, please download the HDF5 from @@ -291,12 +271,10 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); delete gpu_index; -#ifdef CUSTOMIZATION faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } -#endif printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); faiss::write_index(cpu_index, index_file_name.c_str()); @@ -372,15 +350,12 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key faiss::Index *gpu_index, *index; if (query_mode != MODE_CPU) { faiss::gpu::GpuClonerOptions option; -#ifdef CUSTOMIZATION option.allInGpu = true; faiss::IndexComposition index_composition; index_composition.index = cpu_index; index_composition.quantizer = nullptr; -#endif switch (query_mode) { -#ifdef CUSTOMIZATION case MODE_MIX: { index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data @@ -403,9 +378,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key index = cpu_index; break; } -#endif case MODE_GPU: -#ifdef CUSTOMIZATION +#if 1 index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data // warm up the transmission @@ -571,14 +545,12 @@ TEST(FAISSTEST, BENCHMARK) { test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -#ifdef CUSTOMIZATION test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -#endif /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const int32_t GLOVE_INSERT_LOOPS = 1; @@ -589,12 +561,10 @@ TEST(FAISSTEST, BENCHMARK) { test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -#ifdef CUSTOMIZATION test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -#endif } diff --git a/core/src/index/unittest/faiss_benchmark/faiss_bitset_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_bitset_test.cpp index 13ef4e543f..da4809746d 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_bitset_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_bitset_test.cpp @@ -29,21 +29,16 @@ #include #include #include -#include -#include -#include - #include +#include +#include +#include +#include #include +#include #include #include -#ifdef CUSTOMIZATION -#include -#else -#include -#endif - /***************************************************** * To run this test, please download the HDF5 from * https://support.hdfgroup.org/ftp/HDF5/releases/ @@ -284,12 +279,10 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); delete gpu_index; -#ifdef CUSTOMIZATION faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } -#endif printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); faiss::write_index(cpu_index, index_file_name.c_str()); @@ -381,15 +374,12 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key faiss::Index *gpu_index, *index; if (query_mode != MODE_CPU) { faiss::gpu::GpuClonerOptions option; -#ifdef CUSTOMIZATION option.allInGpu = true; faiss::IndexComposition index_composition; index_composition.index = cpu_index; index_composition.quantizer = nullptr; -#endif switch (query_mode) { -#ifdef CUSTOMIZATION case MODE_MIX: { index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data @@ -412,9 +402,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key index = cpu_index; break; } -#endif case MODE_GPU: -#ifdef CUSTOMIZATION +#if 1 index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data // warm up the transmission diff --git a/core/src/index/unittest/test_customized_index.cpp b/core/src/index/unittest/test_customized_index.cpp index b0ce1dae17..f41de356a8 100644 --- a/core/src/index/unittest/test_customized_index.cpp +++ b/core/src/index/unittest/test_customized_index.cpp @@ -42,7 +42,7 @@ class SingleIndexTest : public DataGen, public TestGpuIndexBase { milvus::knowhere::IVFPtr index_ = nullptr; }; -#ifdef CUSTOMIZATION +#ifdef MILVUS_GPU_VERSION TEST_F(SingleIndexTest, IVFSQHybrid) { assert(!xb.empty()); diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index c2de43fab5..7d85155a63 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -84,9 +84,7 @@ INSTANTIATE_TEST_CASE_P( std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_GPU), std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_GPU), std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_GPU), -#ifdef CUSTOMIZATION std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, milvus::knowhere::IndexMode::MODE_GPU), -#endif #endif std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_CPU), std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_CPU), @@ -255,11 +253,9 @@ TEST_P(IVFTest, clone_test) { } } } - #endif #ifdef MILVUS_GPU_VERSION -#ifdef CUSTOMIZATION TEST_P(IVFTest, gpu_seal_test) { if (index_mode_ != milvus::knowhere::IndexMode::MODE_GPU) { return; @@ -300,8 +296,6 @@ TEST_P(IVFTest, gpu_seal_test) { ASSERT_ANY_THROW(milvus::knowhere::cloner::CopyCpuToGpu(cpu_idx, -1, milvus::knowhere::Config())); } -#endif - TEST_P(IVFTest, invalid_gpu_source) { if (index_mode_ != milvus::knowhere::IndexMode::MODE_GPU) { return; @@ -337,7 +331,6 @@ TEST_P(IVFTest, invalid_gpu_source) { ASSERT_ANY_THROW(index_->Train(base_dataset, invalid_conf)); } -#ifdef CUSTOMIZATION TEST_P(IVFTest, IVFSQHybrid_test) { if (index_type_ != milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H) { return; @@ -360,5 +353,3 @@ TEST_P(IVFTest, IVFSQHybrid_test) { ASSERT_ANY_THROW(index->SetQuantizer(nullptr)); } #endif - -#endif diff --git a/core/src/index/unittest/test_vecindex.cpp b/core/src/index/unittest/test_vecindex.cpp index 287fdb1e62..dc231f6403 100644 --- a/core/src/index/unittest/test_vecindex.cpp +++ b/core/src/index/unittest/test_vecindex.cpp @@ -62,14 +62,12 @@ INSTANTIATE_TEST_CASE_P( std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_GPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_GPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_GPU), -#ifdef CUSTOMIZATION std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8H, milvus::knowhere::IndexMode::MODE_GPU), -#endif #endif std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_CPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_CPU), - std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_CPU) - std::make_tuple(milvus::knowhere::IndexType::INDEX_NSG, milvus::knowhere::IndexMode::MODE_CPU), + std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_CPU), + std::make_tuple(milvus::knowhere::IndexType::INDEX_NSG, milvus::knowhere::IndexMode::MODE_CPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_HNSW, milvus::knowhere::IndexMode::MODE_CPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_SPTAG_KDT_RNT, milvus::knowhere::IndexMode::MODE_CPU), std::make_tuple(milvus::knowhere::IndexType::INDEX_SPTAG_BKT_RNT, milvus::knowhere::IndexMode::MODE_CPU))); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 565a5a642d..7ddea4f5fa 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -123,9 +123,7 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); -#ifdef CUSTOMIZATION pass_list.push_back(std::make_shared()); -#endif pass_list.push_back(std::make_shared()); } #endif diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index e2626a41b0..c7ed1e409b 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -8,6 +8,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. + #ifdef MILVUS_GPU_VERSION #include "scheduler/optimizer/FaissIVFSQ8HPass.h" #include "cache/GpuCacheMgr.h" @@ -23,7 +24,6 @@ namespace scheduler { void FaissIVFSQ8HPass::Init() { -#ifdef CUSTOMIZATION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,12 +38,10 @@ FaissIVFSQ8HPass::Init() { AddGpuEnableListener(); AddGpuSearchThresholdListener(); AddGpuSearchResourcesListener(); -#endif } bool FaissIVFSQ8HPass::Run(const TaskPtr& task) { -#ifdef CUSTOMIZATION if (task->Type() != TaskType::SearchTask) { return false; } @@ -72,7 +70,6 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto label = std::make_shared(res_ptr); task->label() = label; return true; -#endif } } // namespace scheduler diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 97a5ba88ae..047d6ca5b5 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -11,6 +11,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" +#include "db/Types.h" #include "db/Utils.h" #include "db/engine/ExecutionEngine.h" #include "knowhere/index/vector_index/ConfAdapter.h" @@ -20,13 +21,10 @@ #include #ifdef MILVUS_GPU_VERSION - #include - #endif #include -#include #include #include #include @@ -169,7 +167,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { return Status(SERVER_INVALID_INDEX_TYPE, msg); } -#ifndef CUSTOMIZATION +#ifndef MILVUS_GPU_VERSION // special case, hybird index only available in customize faiss library if (engine_type == static_cast(engine::EngineType::FAISS_IVFSQ8H)) { std::string msg = "Unsupported index type: " + std::to_string(index_type); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 18ed4f52fc..e2b5cbbddd 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -348,7 +348,6 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } -#ifdef CUSTOMIZATION #ifdef MILVUS_GPU_VERSION index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish @@ -360,7 +359,6 @@ TEST_F(DBTest, SEARCH_TEST) { stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); } -#endif #endif { // search by specify index file @@ -411,7 +409,6 @@ TEST_F(DBTest, SEARCH_TEST) { } #endif -#ifdef CUSTOMIZATION #ifdef MILVUS_GPU_VERSION // test FAISS_IVFSQ8H optimizer index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; @@ -441,7 +438,6 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } #endif -#endif } TEST_F(DBTest, PRELOADTABLE_TEST) { @@ -747,12 +743,10 @@ TEST_F(DBTest, INDEX_TEST) { ASSERT_FALSE(stat.ok()); fiu_disable("DBImpl.UpdateTableIndexRecursively.fail_update_table_index"); -#ifdef CUSTOMIZATION #ifdef MILVUS_GPU_VERSION index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); -#endif #endif milvus::engine::TableIndex index_out; diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index b920ab8f68..ec92529bbd 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -413,7 +413,7 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(), milvus::SERVER_INVALID_INDEX_TYPE); for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) { -#ifndef CUSTOMIZATION +#ifndef MILVUS_GPU_VERSION if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) { ASSERT_NE(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS); continue;