Add the check of C++ code in the CI progress

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

Makefile

@@ -27,10 +27,22 @@ get-check-deps:
 get-build-deps:
 	@(env bash $(PWD)/scripts/install_deps.sh)
 
+clang-format:
+	@echo "Running $@ check"
+	@(env bash ${PWD}/scripts/run_clang_format.sh internal/core)
+
+generated-proto-go:export protoc:=${PWD}/cmake_build/thirdparty/protobuf/protobuf-build/protoc
+generated-proto-go: build-cpp
+	@(env bash $(PWD)/scripts/proto_gen_go.sh)
+
+check-proto-product: generated-proto-go
+	@(env bash $(PWD)/scripts/check_proto_product.sh)
+
 fmt:
 	@echo "Running $@ check"
-	@GO111MODULE=on gofmt -d cmd/
-	@GO111MODULE=on gofmt -d internal/
+	@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh cmd/
+	@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh internal/
+	@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh test/
 
 #TODO: Check code specifications by golangci-lint
 lint:
@@ -46,7 +58,7 @@ ruleguard:
 	@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./cmd/...
 	@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./test/...
 
-verifiers: get-check-deps fmt lint ruleguard
+verifiers: clang-format get-check-deps fmt lint ruleguard
 
 # Builds various components locally.
 build-go:

build/docker/env/cpu/ubuntu18.04/Dockerfile

@@ -17,7 +17,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 && \
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 clang-format-10 && \
     wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
     apt-get update && apt-get install -y --no-install-recommends \
     g++ gcc gfortran git make ccache libssl-dev zlib1g-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-serialization-dev python3-dev libboost-python-dev libcurl4-openssl-dev libtbb-dev clang-format-7 clang-tidy-7 lcov && \

docker-compose.yml

@@ -25,7 +25,7 @@ services:
     working_dir: "/go/src/github.com/zilliztech/milvus-distributed"
     command: &ubuntu-command >
       /bin/bash -c "
-        make build-cpp && make verifiers && make unittest"
+        make check-proto-product && make verifiers && make unittest"
 
   gdbserver:
     image: ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION}

internal/core/src/segcore/SegmentNaive.cpp

@@ -8,6 +8,7 @@
 #include <knowhere/index/vector_index/adapter/VectorAdapter.h>
 #include <knowhere/index/vector_index/VecIndexFactory.h>
 #include <faiss/utils/distances.h>
+#include "segcore/Reduce.h"
 
 namespace milvus::segcore {
 
@@ -294,43 +295,6 @@ SegmentNaive::QueryImpl(query::QueryDeprecatedPtr query_info, Timestamp timestam
     return Status::OK();
 }
 
-static void
-merge_into(int64_t queries,
-           int64_t topk,
-           float* distances,
-           int64_t* uids,
-           const float* new_distances,
-           const int64_t* new_uids) {
-    for (int64_t qn = 0; qn < queries; ++qn) {
-        auto base = qn * topk;
-        auto src2_dis = distances + base;
-        auto src2_uids = uids + base;
-
-        auto src1_dis = new_distances + base;
-        auto src1_uids = new_uids + base;
-
-        std::vector<float> buf_dis(topk);
-        std::vector<int64_t> buf_uids(topk);
-
-        auto it1 = 0;
-        auto it2 = 0;
-
-        for (auto buf = 0; buf < topk; ++buf) {
-            if (src1_dis[it1] <= src2_dis[it2]) {
-                buf_dis[buf] = src1_dis[it1];
-                buf_uids[buf] = src1_uids[it1];
-                ++it1;
-            } else {
-                buf_dis[buf] = src2_dis[it2];
-                buf_uids[buf] = src2_uids[it2];
-                ++it2;
-            }
-        }
-        std::copy_n(buf_dis.data(), topk, src2_dis);
-        std::copy_n(buf_uids.data(), topk, src2_uids);
-    }
-}
-
 Status
 SegmentNaive::QueryBruteForceImpl(query::QueryDeprecatedPtr query_info, Timestamp timestamp, QueryResult& results) {
     auto ins_barrier = get_barrier(record_, timestamp);

internal/core/unittest/test_indexing.cpp

@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <chrono>
 #include "test_utils/Timer.h"
+#include "segcore/Reduce.h"
 
 using std::cin;
 using std::cout;
@@ -53,43 +54,6 @@ generate_data(int N) {
 }
 }  // namespace
 
-void
-merge_into(int64_t queries,
-           int64_t topk,
-           float* distances,
-           int64_t* uids,
-           const float* new_distances,
-           const int64_t* new_uids) {
-    for (int64_t qn = 0; qn < queries; ++qn) {
-        auto base = qn * topk;
-        auto src2_dis = distances + base;
-        auto src2_uids = uids + base;
-
-        auto src1_dis = new_distances + base;
-        auto src1_uids = new_uids + base;
-
-        std::vector<float> buf_dis(topk);
-        std::vector<int64_t> buf_uids(topk);
-
-        auto it1 = 0;
-        auto it2 = 0;
-
-        for (auto buf = 0; buf < topk; ++buf) {
-            if (src1_dis[it1] <= src2_dis[it2]) {
-                buf_dis[buf] = src1_dis[it1];
-                buf_uids[buf] = src1_uids[it1];
-                ++it1;
-            } else {
-                buf_dis[buf] = src2_dis[it2];
-                buf_uids[buf] = src2_uids[it2];
-                ++it2;
-            }
-        }
-        std::copy_n(buf_dis.data(), topk, src2_dis);
-        std::copy_n(buf_uids.data(), topk, src2_uids);
-    }
-}
-
 TEST(Indexing, SmartBruteForce) {
     // how to ?
     // I'd know

scripts/check_proto_product.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+cd ${SCRIPTS_DIR}
+
+GO_SRC_DIR="${SCRIPTS_DIR}/$1"
+if test -z "$(git status | grep -E "*pb.go|*pb.cc|*pb.h")"; then
+  exit 0
+else
+  echo "The go file or cpp file generated by proto are not latest!"
+  exit 1
+fi

scripts/gofmt.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+GO_SRC_DIR="${SCRIPTS_DIR}/$1"
+if test -z "$(gofmt -d $GO_SRC_DIR)"; then
+  exit 0
+else
+  gofmt -d $GO_SRC_DIR
+  echo "Please format your code by gofmt!"
+  exit 1
+fi

scripts/run_clang_format.sh

@@ -0,0 +1,13 @@
+if [ -z $1 ]; then
+    echo "usage: $0 <path_to_core>"
+    exit -1
+fi
+CorePath=$1
+
+formatThis() {
+    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "/thirdparty" | grep -v "\.pb\." | xargs clang-format-10 -i
+}
+
+formatThis "${CorePath}/src"
+formatThis "${CorePath}/unittest"
+