Fix#1484 (#1490)

* update

Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

* #1426 Support to configure whether to enabled autoflush (#1468)

Signed-off-by: shengjun.li <shengjun.li@zilliz.com>
Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

* fix cases: test_search_id/test_wal (#1486)

Signed-off-by: del-zhenwu <zw@zilliz.com>
Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

* #1480 add return code for AVX512 selection (#1482)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

* update

Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

* update

Signed-off-by: Zhiru Zhu <zzhu@fandm.edu>

Co-authored-by: shengjun.li <49774184+shengjun1985@users.noreply.github.com>
Co-authored-by: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com>
Co-authored-by: Cai Yudong <yudong.cai@zilliz.com>

CHANGELOG.md

@@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1298 Unittest failed when on CPU2GPU case
 -   \#1359 Negative distance value returned when searching with HNSW index type
 -   \#1429 Server crashed when searching vectors using GPU
+-   \#1484 Index type changed to IDMAP after compacted 
 
 ## Feature
 -   \#216 Add CLI to get server info

core/src/db/DBImpl.cpp

@@ -658,12 +658,23 @@ DBImpl::Compact(const std::string& table_id) {
 
     const std::lock_guard<std::mutex> lock(flush_merge_compact_mutex_);
 
+    // Save table index
+    TableIndex table_index;
+    status = DescribeIndex(table_id, table_index);
+    if (!status.ok()) {
+        return status;
+    }
+
     // Drop all index
     status = DropIndex(table_id);
     if (!status.ok()) {
-        std::string err_msg = "Failed to drop index in compact: " + status.message();
-        ENGINE_LOG_ERROR << err_msg;
-        return Status(DB_ERROR, err_msg);
+        return status;
+    }
+
+    // Then update table index to the previous index
+    status = UpdateTableIndexRecursively(table_id, table_index);
+    if (!status.ok()) {
+        return status;
     }
 
     // Get files to compact from meta.
@@ -679,23 +690,51 @@ DBImpl::Compact(const std::string& table_id) {
     ENGINE_LOG_DEBUG << "Found " << files_to_compact.size() << " segment to compact";
 
     OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact);
-    for (auto& file : files_to_compact) {
-        status = CompactFile(table_id, file);
 
-        if (!status.ok()) {
-            OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);
-            return status;
+    meta::TableFilesSchema files_to_update;
+    Status compact_status;
+    for (auto& file : files_to_compact) {
+        compact_status = CompactFile(table_id, file, files_to_update);
+
+        if (!compact_status.ok()) {
+            ENGINE_LOG_ERROR << "Compact failed for file " << file.file_id_ << ": " << compact_status.message();
+            break;
         }
     }
+
+    if (compact_status.ok()) {
+        ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
+    }
+
+    ENGINE_LOG_ERROR << "Updating meta after compaction...";
+
+    // Drop index again, in case some files were in the index building process during compacting
+    status = DropIndex(table_id);
+    if (!status.ok()) {
+        return status;
+    }
+
+    // Update index
+    status = UpdateTableIndexRecursively(table_id, table_index);
+    if (!status.ok()) {
+        return status;
+    }
+
+    status = meta_ptr_->UpdateTableFiles(files_to_update);
+    if (!status.ok()) {
+        return status;
+    }
+
     OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);
 
-    ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
+    ENGINE_LOG_DEBUG << "Finished updating meta after compaction";
 
     return status;
 }
 
 Status
-DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file) {
+DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& file,
+                    meta::TableFilesSchema& files_to_update) {
     ENGINE_LOG_DEBUG << "Compacting segment " << file.segment_id_ << " for table: " << table_id;
 
     // Create new table file
@@ -740,10 +779,6 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab
         return status;
     }
 
-    // Drop index again, in case some files were in the index building process during merging
-    // TODO: might be too frequent?
-    DropIndex(table_id);
-
     // Update table files state
     // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
     // else set file type to RAW, no need to build index
@@ -763,7 +798,10 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab
     }
 
     updated.emplace_back(compacted_file);
-    status = meta_ptr_->UpdateTableFiles(updated);
+
+    for (auto& f : updated) {
+        files_to_update.emplace_back(f);
+    }
 
     ENGINE_LOG_DEBUG << "Compacted segment " << compacted_file.segment_id_ << " from "
                      << std::to_string(file_to_compact.file_size_) << " bytes to "

core/src/db/DBImpl.h

@@ -182,7 +182,8 @@ class DBImpl : public DB {
     BackgroundBuildIndex();
 
     Status
-    CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file);
+    CompactFile(const std::string& table_id, const meta::TableFileSchema& file,
+                meta::TableFilesSchema& files_to_update);
 
     /*
     Status

core/unittest/db/test_delete.cpp

@@ -571,7 +571,11 @@ TEST_F(DeleteTest, compact_with_index) {
 
     stat = db_->Compact(GetTableName());
     ASSERT_TRUE(stat.ok());
-    // std::this_thread::sleep_for(std::chrono::seconds(5));  // wait for build index to finish
+
+    milvus::engine::TableIndex table_index;
+    stat = db_->DescribeIndex(GetTableName(), table_index);
+    ASSERT_TRUE(stat.ok());
+    ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_);
 
     int topk = 10, nprobe = 10;
     for (auto& pair : search_vectors) {