Return status and add try catch exception in index builder

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

internal/core/CMakeLists.txt

@@ -214,3 +214,9 @@ install(
 
 install(FILES ${CMAKE_BINARY_DIR}/src/indexbuilder/libmilvus_indexbuilder.so
         DESTINATION lib)
+
+install(
+        DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/common/
+        DESTINATION include/common/
+        FILES_MATCHING PATTERN "*_c.h"
+)

internal/core/src/common/status_c.h

@@ -0,0 +1,30 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum ErrorCode {
+    Success = 0,
+    UnexpectedException = 1,
+};
+
+typedef struct CStatus {
+    int error_code;
+    const char* error_msg;
+} CStatus;
+
+#ifdef __cplusplus
+}
+#endif

internal/core/src/indexbuilder/index_c.cpp

@@ -26,11 +26,27 @@ class CGODebugUtils {
     }
 };
 
-CIndex
+CStatus
-CreateIndex(const char* serialized_type_params, const char* serialized_index_params) {
+CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index) {
-    auto index = std::make_unique<milvus::indexbuilder::IndexWrapper>(serialized_type_params, serialized_index_params);
+    auto status = CStatus();
-
+    try {
-    return index.release();
+        //    std::cout << "strlen(serialized_type_params): " << CGODebugUtils::Strlen(serialized_type_params,
+        //    type_params_size)
+        //              << std::endl;
+        //    std::cout << "type_params_size: " << type_params_size << std::endl;
+        //    std::cout << "strlen(serialized_index_params): "
+        //              << CGODebugUtils::Strlen(serialized_index_params, index_params_size) << std::endl;
+        //    std::cout << "index_params_size: " << index_params_size << std::endl;
+        auto index =
+            std::make_unique<milvus::indexbuilder::IndexWrapper>(serialized_type_params, serialized_index_params);
+        *res_index = index.release();
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::runtime_error& e) {
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
 }
 
 void
@@ -39,30 +55,57 @@ DeleteIndex(CIndex index) {
     delete cIndex;
 }
 
-void
+CStatus
 BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors) {
-    auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
+    auto status = CStatus();
-    auto dim = cIndex->dim();
+    try {
-    auto row_nums = float_value_num / dim;
+        auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
-    auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
+        auto dim = cIndex->dim();
-    cIndex->BuildWithoutIds(ds);
+        auto row_nums = float_value_num / dim;
+        auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
+        cIndex->BuildWithoutIds(ds);
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::runtime_error& e) {
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
 }
 
-void
+CStatus
 BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors) {
-    auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
+    auto status = CStatus();
-    auto dim = cIndex->dim();
+    try {
-    auto row_nums = (data_size * 8) / dim;
+        auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
-    auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
+        auto dim = cIndex->dim();
-    cIndex->BuildWithoutIds(ds);
+        auto row_nums = (data_size * 8) / dim;
+        auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
+        cIndex->BuildWithoutIds(ds);
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::runtime_error& e) {
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
 }
 
-char*
+CStatus
-SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size) {
+SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer) {
-    auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
+    auto status = CStatus();
-    auto binary = cIndex->Serialize();
+    try {
-    *buffer_size = binary.size;
+        auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
-    return binary.data;
+        auto binary = cIndex->Serialize();
+        *buffer_size = binary.size;
+        *res_buffer = binary.data;
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::runtime_error& e) {
+        status.error_code = UnexpectedException;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
 }
 
 void

internal/core/src/indexbuilder/index_c.h

@@ -28,26 +28,27 @@ extern "C" {
 
 #include <stdint.h>
 #include "segcore/collection_c.h"
+#include "common/status_c.h"
 
 typedef void* CIndex;
 
 // TODO: how could we pass map between go and c++ more efficiently?
 // Solution: using protobuf instead of json, this way significantly increase programming efficiency
 
-CIndex
+CStatus
-CreateIndex(const char* serialized_type_params, const char* serialized_index_params);
+CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index);
 
 void
 DeleteIndex(CIndex index);
 
-void
+CStatus
 BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
 
-void
+CStatus
 BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors);
 
-char*
+CStatus
-SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
+SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer);
 
 void
 LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size);

internal/core/src/segcore/collection_c.h

@@ -15,16 +15,6 @@
 extern "C" {
 #endif
 
-enum ErrorCode {
-    Success = 0,
-    UnexpectedException = 1,
-};
-
-typedef struct CStatus {
-    int error_code;
-    const char* error_msg;
-} CStatus;
-
 typedef void* CCollection;
 
 CCollection

internal/core/src/segcore/load_index_c.h

@@ -18,6 +18,7 @@ extern "C" {
 #include <stdint.h>
 
 #include "segcore/collection_c.h"
+#include "common/status_c.h"
 
 typedef void* CLoadIndexInfo;
 typedef void* CBinarySet;

internal/core/src/segcore/plan_c.h

@@ -16,6 +16,7 @@ extern "C" {
 #include <stdbool.h>
 #include <stdint.h>
 #include "segcore/collection_c.h"
+#include "common/status_c.h"
 
 typedef void* CPlan;
 typedef void* CPlaceholderGroup;

internal/core/src/segcore/reduce_c.h

@@ -16,6 +16,7 @@ extern "C" {
 #include <stdbool.h>
 #include <stdint.h>
 #include "segcore/segment_c.h"
+#include "common/status_c.h"
 
 typedef void* CMarshaledHits;
 

internal/core/src/segcore/segment_c.h

@@ -19,6 +19,7 @@ extern "C" {
 
 #include "segcore/plan_c.h"
 #include "segcore/load_index_c.h"
+#include "common/status_c.h"
 
 typedef void* CSegmentBase;
 typedef void* CQueryResult;

internal/indexbuilder/index.go

@@ -13,6 +13,8 @@ package indexbuilder
 */
 import "C"
 import (
+	"errors"
+	"strconv"
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
@@ -38,11 +40,20 @@ type CIndex struct {
 
 func (index *CIndex) Serialize() ([]*Blob, error) {
 	/*
-		char*
+		CStatus
-		SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
+		SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer);
 	*/
+
+	var cDumpedSlicedBuffer *C.char
 	var bufferSize int32
-	var cDumpedSlicedBuffer *C.char = C.SerializeToSlicedBuffer(index.indexPtr, (*C.int32_t)(unsafe.Pointer(&bufferSize)))
+	status := C.SerializeToSlicedBuffer(index.indexPtr, (*C.int32_t)(unsafe.Pointer(&bufferSize)), &cDumpedSlicedBuffer)
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return nil, errors.New("SerializeToSlicedBuffer failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+
 	defer C.free(unsafe.Pointer(cDumpedSlicedBuffer))
 
 	dumpedSlicedBuffer := C.GoBytes(unsafe.Pointer(cDumpedSlicedBuffer), (C.int32_t)(bufferSize))
@@ -81,19 +92,31 @@ func (index *CIndex) Load(blobs []*Blob) error {
 
 func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error {
 	/*
-		void
+		CStatus
 		BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
 	*/
-	C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
+	status := C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New("BuildFloatVecIndexWithoutIds failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
 	return nil
 }
 
 func (index *CIndex) BuildBinaryVecIndexWithoutIds(vectors []byte) error {
 	/*
-		void
+		CStatus
 		BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors);
 	*/
-	C.BuildBinaryVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0]))
+	status := C.BuildBinaryVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0]))
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
 	return nil
 }
 
@@ -127,11 +150,21 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
 	indexParamsPointer := C.CString(indexParamsStr)
 
 	/*
-		CIndex
+		CStatus
 		CreateIndex(const char* serialized_type_params,
-					const char* serialized_index_params);
+					const char* serialized_index_params,
+					CIndex* res_index);
 	*/
+	var indexPtr C.CIndex
+	status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr)
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return nil, errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+
 	return &CIndex{
-		indexPtr: C.CreateIndex(typeParamsPointer, indexParamsPointer),
+		indexPtr: indexPtr,
 	}, nil
 }