Add cgo building script and init ago insertion interface

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

core/CMakeLists.txt

@@ -9,3 +9,12 @@ set (CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 include_directories(src)
 add_subdirectory(src)
 add_subdirectory(unittest)
+
+install(
+        DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/dog_segment/
+        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/include
+        FILES_MATCHING PATTERN "*_c.h"
+)
+
+install(FILES ${CMAKE_BINARY_DIR}/src/dog_segment/libmilvus_dog_segment.so
+        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/lib)

core/build.sh

@@ -0,0 +1,8 @@
+if [[ -d "./build" ]]
+then
+    rm -rf build
+fi
+
+mkdir build && cd build
+cmake ..
+make -j8 && make install

core/include/segment_c.h

@@ -10,7 +10,7 @@ extern "C" {
 
 typedef void* CSegmentBase;
 
-CSegmentBase SegmentBaseInit();
+CSegmentBase SegmentBaseInit(unsigned long segment_id);
 
 //int32_t Insert(CSegmentBase c_segment, signed long int size, const unsigned long* primary_keys, const unsigned long int* timestamps, DogDataChunk values);
 

core/src/dog_segment/CMakeLists.txt

@@ -1,7 +1,10 @@
 set(DOG_SEGMENT_FILES
         SegmentNaive.cpp
         Collection.cpp
-        cwrap.cpp
+        Partition.cpp
+        collection_c.cpp
+        partition_c.cpp
+        segment_c.cpp
         )
 # Third Party dablooms file
 #aux_source_directory( ${MILVUS_THIRDPARTY_SRC}/dablooms  THIRDPARTY_DABLOOMS_FILES )
@@ -11,7 +14,3 @@ add_library(milvus_dog_segment SHARED
 #add_dependencies( segment sqlite mysqlpp )
 
 target_link_libraries(milvus_dog_segment tbb milvus_utils pthread)
-
-#add_executable(main main.cpp)
-#target_link_libraries(main
-#        milvus_dog_segment)

core/src/dog_segment/Collection.cpp

@@ -1,2 +1,17 @@
 #include "Collection.h"
 
+namespace milvus::dog_segment {
+
+Collection::Collection(std::string &collection_name, std::string &schema):
+      collection_name_(collection_name), schema_json_(schema){}
+
+void
+Collection::set_index() {}
+
+void
+Collection::parse() {}
+
+void
+Collection::AddNewPartition() {}
+
+}

core/src/dog_segment/Collection.h

@@ -1,52 +1,21 @@
 #pragma once
 
+#include "dog_segment/Partition.h"
 #include "SegmentDefs.h"
-#include "SegmentBase.h"
-
-//////////////////////////////////////////////////////////////////
 
 namespace milvus::dog_segment {
 
-class Partition {
-public:
-    explicit Partition(std::string& partition_name): partition_name_(partition_name) {}
-
-    const std::vector<SegmentBasePtr> &segments() const {
-      return segments_;
-    }
-
-private:
-    std::string partition_name_;
-    std::vector<SegmentBasePtr> segments_;
-};
-
-using PartitionPtr = std::shared_ptr<Partition>;
-
-//////////////////////////////////////////////////////////////////
-
 class Collection {
 public:
-    explicit Collection(std::string &collection_name, std::string &schema)
-        : collection_name_(collection_name), schema_json_(schema) {}
+    explicit Collection(std::string &collection_name, std::string &schema);
 
     // TODO: set index
-    void set_index() {}
+    void set_index();
 
-    void parse() {
-      // TODO: config to schema
-    }
+    // TODO: config to schema
+    void parse();
 
-public:
-
-//    std::vector<int64_t> Insert() {
-//       for (auto partition: partitions_) {
-//         for (auto segment: partition.segments()) {
-//           if (segment.Status == Status.open) {
-//             segment.Insert()
-//           }
-//         }
-//       }
-//    }
+    void AddNewPartition();
 
 private:
     // TODO: add Index ptr

core/src/dog_segment/Partition.cpp

@@ -0,0 +1,20 @@
+#include "Partition.h"
+
+namespace milvus::dog_segment {
+
+Partition::Partition(std::string& partition_name):
+      partition_name_(partition_name) {}
+
+void
+Partition::AddNewSegment(uint64_t segment_id) {
+  auto segment = CreateSegment();
+  segment->set_segment_id(segment_id);
+  segments_.emplace_back(segment);
+}
+
+Partition*
+CreatePartition() {
+
+}
+
+}

core/src/dog_segment/Partition.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "SegmentBase.h"
+
+namespace milvus::dog_segment {
+
+class Partition {
+public:
+    explicit Partition(std::string& partition_name);
+
+    const std::vector<SegmentBasePtr> &segments() const {
+      return segments_;
+    }
+
+    void AddNewSegment(uint64_t segment_id);
+
+private:
+    std::string partition_name_;
+    std::vector<SegmentBasePtr> segments_;
+};
+
+using PartitionPtr = std::shared_ptr<Partition>;
+
+Partition* CreatePartiton();
+
+}

core/src/dog_segment/SegmentNaive.cpp

@@ -171,7 +171,6 @@ SegmentNaive::Insert(int64_t size, const uint64_t* primary_keys, const Timestamp
     const auto& schema = *schema_;
     auto data_chunk = ColumnBasedDataChunk::from(row_values, schema);
 
-    std::cout << "key:" << std::endl;
     // insert datas
     // TODO: use shared_lock
     std::lock_guard lck(mutex_);
@@ -180,7 +179,6 @@ SegmentNaive::Insert(int64_t size, const uint64_t* primary_keys, const Timestamp
     uids_.grow_by(primary_keys, primary_keys + size);
     for(int64_t i = 0; i < size; ++i) {
         auto key = primary_keys[i];
-        std::cout << key << std::endl;
         auto internal_index = i + ack_id;
         internal_indexes_[key] = internal_index;
     }

core/src/dog_segment/collection_c.cpp

@@ -0,0 +1,11 @@
+#include "collection_c.h"
+
+CCollection
+NewCollection(const char* collection_name) {
+
+}
+
+void
+DeleteCollection(CCollection collection) {
+
+}

core/src/dog_segment/collection_c.h

@@ -0,0 +1,13 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* CCollection;
+
+CCollection NewCollection(const char* collection_name);
+
+void DeleteCollection(CCollection collection);
+
+#ifdef __cplusplus
+}
+#endif

core/src/dog_segment/main.cpp

@@ -1,67 +0,0 @@
-#include <vector>
-#include <cstdint>
-#include <random>
-#include <iostream>
-#include "cwrap.h"
-
-//int main() {
-//  auto s = SegmentBaseInit();
-//
-//  std::vector<char> raw_data;
-//  std::vector<uint64_t> timestamps;
-//  std::vector<uint64_t> uids;
-//  int N = 10000;
-//  std::default_random_engine e(67);
-//  for(int i = 0; i < N; ++i) {
-//    uids.push_back(100000 + i);
-//    timestamps.push_back(0);
-//    // append vec
-//    float vec[16];
-//    for(auto &x: vec) {
-//      x = e() % 2000 * 0.001 - 1.0;
-//    }
-//    raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
-//    int age = e() % 100;
-//    raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
-//  }
-//
-//  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
-//
-//  DogDataChunk dogDataChunk{};
-//  dogDataChunk.count = N;
-//  dogDataChunk.raw_data = raw_data.data();
-//  dogDataChunk.sizeof_per_row = (int)line_sizeof;
-//
-//  auto res = Insert(s, N, uids.data(), timestamps.data(), dogDataChunk);
-//
-//  std::cout << res << std::endl;
-//}
-
-int main() {
-  auto s = SegmentBaseInit();
-
-  std::vector<char> raw_data;
-  std::vector<uint64_t> timestamps;
-  std::vector<uint64_t> uids;
-  int N = 10000;
-  std::default_random_engine e(67);
-  for(int i = 0; i < N; ++i) {
-    uids.push_back(100000 + i);
-    timestamps.push_back(0);
-    // append vec
-    float vec[16];
-    for(auto &x: vec) {
-      x = e() % 2000 * 0.001 - 1.0;
-    }
-    raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
-    int age = e() % 100;
-    raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
-  }
-
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
-
-  auto res = Insert(s, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
-
-  std::cout << res << std::endl;
-}
-

core/src/dog_segment/partition_c.cpp

@@ -0,0 +1,12 @@
+#include "partition_c.h"
+#include "Partition.h"
+#include "Collection.h"
+
+CPartition
+NewPartition(CCollection collection, const char* partition_name) {
+  auto name = std::string(partition_name);
+  auto partition = new milvus::dog_segment::Partition(name);
+
+  auto co = (milvus::dog_segment::Collection*)collection;
+  co->AddNewPartition();
+}

core/src/dog_segment/partition_c.h

@@ -0,0 +1,15 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "collection_c.h"
+
+typedef void* CPartition;
+
+CPartition NewPartition(CCollection collection, const char* partition_name);
+
+void DeletePartition(CPartition partition);
+
+#ifdef __cplusplus
+}
+#endif

core/src/dog_segment/segment_c.cpp

@@ -1,10 +1,11 @@
 #include "SegmentBase.h"
-#include "cwrap.h"
+#include "segment_c.h"
 
 CSegmentBase
-SegmentBaseInit() {
+SegmentBaseInit(unsigned long segment_id) {
   std::cout << "Hello milvus" << std::endl;
   auto seg = milvus::dog_segment::CreateSegment();
+  seg->set_segment_id(segment_id);
   return (void*)seg;
 }
 

core/src/dog_segment/segment_c.h

@@ -0,0 +1,27 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//struct DogDataChunk {
+//    void* raw_data;      // schema
+//    int sizeof_per_row;  // alignment
+//    signed long int count;
+//};
+
+typedef void* CSegmentBase;
+
+CSegmentBase SegmentBaseInit(unsigned long segment_id);
+
+//int32_t Insert(CSegmentBase c_segment, signed long int size, const unsigned long* primary_keys, const unsigned long int* timestamps, DogDataChunk values);
+
+int Insert(CSegmentBase c_segment,
+                signed long int size,
+                const unsigned long* primary_keys,
+                const unsigned long int* timestamps,
+                void* raw_data,
+                int sizeof_per_row,
+                signed long int count);
+
+#ifdef __cplusplus
+}
+#endif

core/unittest/CMakeLists.txt

@@ -2,6 +2,7 @@ enable_testing()
 find_package(GTest REQUIRED)
 set(MILVUS_TEST_FILES
     test_dog_segment.cpp
+        test_c_api.cpp
 )
 add_executable(all_tests 
     ${MILVUS_TEST_FILES}

core/unittest/test_c_api.cpp

@@ -0,0 +1,36 @@
+#include <iostream>
+#include <string>
+#include <random>
+#include <gtest/gtest.h>
+
+#include "dog_segment/segment_c.h"
+#include "dog_segment/collection_c.h"
+
+TEST(SegmentTest, InsertTest) {
+  auto segment_id = 0;
+  auto s = SegmentBaseInit(segment_id);
+
+  std::vector<char> raw_data;
+  std::vector<uint64_t> timestamps;
+  std::vector<uint64_t> uids;
+  int N = 10000;
+  std::default_random_engine e(67);
+  for(int i = 0; i < N; ++i) {
+    uids.push_back(100000 + i);
+    timestamps.push_back(0);
+    // append vec
+    float vec[16];
+    for(auto &x: vec) {
+      x = e() % 2000 * 0.001 - 1.0;
+    }
+    raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
+    int age = e() % 100;
+    raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
+  }
+
+  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
+
+  auto res = Insert(s, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
+
+  std::cout << res << std::endl;
+}

reader/query_node.go

@@ -119,7 +119,8 @@ func (node *QueryNode) InitQueryNodeCollection() {
 	node.Collections = append(node.Collections, collection)
 	var partition, _ = collection.NewPartition("partition1")
 	collection.Partitions = append(collection.Partitions, partition)
-	var segment, _ = partition.NewSegment()
+	// TODO: add segment id
+	var segment, _ = partition.NewSegment(0)
 	partition.Segments = append(partition.Segments, segment)
 }
 
@@ -128,10 +129,11 @@ func (node *QueryNode) SegmentsManagement() {
 	for _, collection := range node.Collections {
 		for _, partition := range collection.Partitions {
 			for _, segment := range partition.Segments {
+				// TODO: check segment status
 				if timeSync >= segment.SegmentCloseTime {
 					segment.Close()
 					// TODO: add atomic segment id
-					var newSegment, _ = partition.NewSegment()
+					var newSegment, _ = partition.NewSegment(0)
 					newSegment.SegmentCloseTime = timeSync + SegmentLifetime
 					partition.Segments = append(partition.Segments, newSegment)
 				}

reader/segment.go

@@ -1,5 +1,14 @@
 package reader
 
+/*
+
+#cgo CFLAGS: -I../core/include
+
+#cgo LDFLAGS: -L../core/lib -lmilvus_dog_segment -Wl,-rpath=../core/lib
+
+#include "segment_c.h"
+
+*/
 import "C"
 import (
 	"errors"
@@ -14,9 +23,9 @@ type Segment struct {
 	SegmentCloseTime uint64
 }
 
-func (p *Partition) NewSegment() (*Segment, error) {
+func (p *Partition) NewSegment(segmentId uint64) (*Segment, error) {
 	// TODO: add segment id
-	segmentPtr, status := C.CreateSegment(p.PartitionPtr)
+	segmentPtr, status := C.SegmentBaseInit(p.PartitionPtr)
 
 	if status != 0 {
 		return nil, errors.New("create segment failed")

reader/test/cgo_segment_test.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestIntMinBasic(t *testing.T) {
+	fmt.Println("Hello go testing")
+}