From 1927eaf88bd6b405304ce53fa13e8fe962414463 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 31 Dec 2020 16:38:37 +0800 Subject: [PATCH] Add revive linter Signed-off-by: quicksilver --- Makefile | 12 +- internal/core/src/segcore/segment_c.cpp | 4 +- internal/core/unittest/test_c_api.cpp | 461 +++++++++++++++++- internal/querynode/load_index_service_test.go | 17 - scripts/gofmt.sh | 2 +- tools/check/go.mod | 14 + tools/check/go.sum | 93 ++++ tools/check/revive.toml | 30 ++ 8 files changed, 608 insertions(+), 25 deletions(-) create mode 100644 tools/check/go.mod create mode 100644 tools/check/go.sum create mode 100644 tools/check/revive.toml diff --git a/Makefile b/Makefile index 5812544722..1ae87e6930 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,10 @@ getdeps: @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) @which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...) +tools/bin/revive: tools/check/go.mod + cd tools/check; \ + $(GO) build -o ../bin/revive github.com/mgechev/revive + cppcheck: @(env bash ${PWD}/scripts/core_build.sh -l) @@ -49,8 +53,12 @@ else @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh tests/go/ endif +lint:tools/bin/revive + @echo "Running $@ check" + @tools/bin/revive -formatter friendly -config tools/check/revive.toml ./... + #TODO: Check code specifications by golangci-lint -lint: +static-check: @echo "Running $@ check" @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/... @@ -68,7 +76,7 @@ else @${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./tests/go/... endif -verifiers: getdeps cppcheck fmt lint ruleguard +verifiers: getdeps cppcheck fmt static-check ruleguard # Builds various components locally. build-go: build-cpp diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index 182301c7b7..2bed416f93 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -181,15 +181,15 @@ FillTargetEntry(CSegmentBase c_segment, CPlan c_plan, CQueryResult c_result) { CStatus UpdateSegmentIndex(CSegmentBase c_segment, CLoadIndexInfo c_load_index_info) { + auto status = CStatus(); try { auto segment = (milvus::segcore::SegmentBase*)c_segment; auto load_index_info = (LoadIndexInfo*)c_load_index_info; - auto status = CStatus(); + auto res = segment->LoadIndexing(*load_index_info); status.error_code = Success; status.error_msg = ""; return status; } catch (std::exception& e) { - auto status = CStatus(); status.error_code = UnexpectedException; status.error_msg = strdup(e.what()); return status; diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index dedfd71c29..a24770e037 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "pb/service_msg.pb.h" #include "segcore/reduce_c.h" @@ -23,9 +24,17 @@ #include #include #include +#include +#include +#include "test_utils/DataGen.h" namespace chrono = std::chrono; +using namespace milvus; +using namespace milvus::segcore; +using namespace milvus::proto; +using namespace milvus::knowhere; + TEST(CApiTest, CollectionTest) { auto schema_tmp_conf = ""; auto collection = NewCollection(schema_tmp_conf); @@ -344,11 +353,11 @@ TEST(CApiTest, GetMemoryUsageInBytesTest) { namespace { auto generate_data(int N) { - std::vector raw_data; + std::vector raw_data; std::vector timestamps; std::vector uids; std::default_random_engine er(42); - std::uniform_real_distribution<> distribution(0.0, 1.0); + std::normal_distribution<> distribution(0.0, 1.0); std::default_random_engine ei(42); for (int i = 0; i < N; ++i) { uids.push_back(10 * N + i); @@ -358,12 +367,58 @@ generate_data(int N) { for (auto& x : vec) { x = distribution(er); } - raw_data.insert(raw_data.end(), std::begin(vec), std::end(vec)); + raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); int age = ei() % 100; raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); } return std::make_tuple(raw_data, timestamps, uids); } + +std::string +generate_collection_shema(std::string metric_type, std::string dim) { + schema::CollectionSchema collection_schema; + collection_schema.set_name("collection_test"); + collection_schema.set_autoid(true); + + auto vec_field_schema = collection_schema.add_fields(); + vec_field_schema->set_name("fakevec"); + vec_field_schema->set_fieldid(0); + vec_field_schema->set_data_type(schema::DataType::VECTOR_FLOAT); + auto metric_type_param = vec_field_schema->add_index_params(); + metric_type_param->set_key("metric_type"); + metric_type_param->set_value(metric_type); + auto dim_param = vec_field_schema->add_type_params(); + dim_param->set_key("dim"); + dim_param->set_value(dim); + + auto other_field_schema = collection_schema.add_fields(); + ; + other_field_schema->set_name("counter"); + other_field_schema->set_fieldid(1); + other_field_schema->set_data_type(schema::DataType::INT64); + + std::string schema_string; + auto marshal = google::protobuf::TextFormat::PrintToString(collection_schema, &schema_string); + assert(marshal == true); + return schema_string; +} + +VecIndexPtr +generate_index( + void* raw_data, milvus::knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) { + auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type, knowhere::IndexMode::MODE_CPU); + + auto database = milvus::knowhere::GenDataset(N, dim, raw_data); + indexing->Train(database, conf); + indexing->AddWithoutIds(database, conf); + EXPECT_EQ(indexing->Count(), N); + EXPECT_EQ(indexing->Dim(), dim); + + EXPECT_EQ(indexing->Count(), N); + EXPECT_EQ(indexing->Dim(), dim); + return indexing; +} + } // namespace // TEST(CApiTest, TestSearchPreference) { @@ -776,4 +831,404 @@ TEST(CApiTest, LoadIndex_Search) { for (int i = 0; i < std::min(num_query * K, 100); ++i) { std::cout << ids[i] << "->" << dis[i] << std::endl; } +} + +TEST(CApiTest, UpdateSegmentIndex_Without_Predicate) { + // insert data to segment + constexpr auto DIM = 16; + constexpr auto K = 5; + + std::string schema_string = generate_collection_shema("L2", "16"); + auto collection = NewCollection(schema_string.c_str()); + auto schema = ((segcore::Collection*)collection)->get_schema(); + auto segment = NewSegment(collection, 0); + + auto N = 1000 * 1000; + auto dataset = DataGen(schema, N); + auto vec_col = dataset.get_col(0); + auto query_ptr = vec_col.data() + 420000 * DIM; + + PreInsert(segment, N); + auto ins_res = Insert(segment, 0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data, + dataset.raw_.sizeof_per_row, dataset.raw_.count); + assert(ins_res.error_code == Success); + + const char* dsl_string = R"( + { + "bool": { + "vector": { + "fakevec": { + "metric_type": "L2", + "params": { + "nprobe": 10 + }, + "query": "$0", + "topk": 5 + } + } + } + })"; + + // create place_holder_group + int num_queries = 5; + auto raw_group = CreatePlaceholderGroupFromBlob(num_queries, 16, query_ptr); + auto blob = raw_group.SerializeAsString(); + + // search on segment's small index + void* plan = nullptr; + auto status = CreatePlan(collection, dsl_string, &plan); + assert(status.error_code == Success); + + void* placeholderGroup = nullptr; + status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup); + assert(status.error_code == Success); + + std::vector placeholderGroups; + placeholderGroups.push_back(placeholderGroup); + Timestamp time = 10000000; + + CQueryResult c_search_result_on_smallIndex; + auto res_before_load_index = + Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_smallIndex); + assert(res_before_load_index.error_code == Success); + + // load index to segment + auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, + {milvus::knowhere::meta::TOPK, K}, + {milvus::knowhere::IndexParams::nlist, 100}, + {milvus::knowhere::IndexParams::nprobe, 10}, + {milvus::knowhere::IndexParams::m, 4}, + {milvus::knowhere::IndexParams::nbits, 8}, + {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {milvus::knowhere::meta::DEVICEID, 0}}; + auto indexing = generate_index(vec_col.data(), conf, DIM, K, N, IndexEnum::INDEX_FAISS_IVFPQ); + + // gen query dataset + auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto result_on_index = indexing->Query(query_dataset, conf, nullptr); + auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); + auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + std::vector vec_ids(ids, ids + K * num_queries); + std::vector vec_dis; + for (int j = 0; j < K * num_queries; ++j) { + vec_dis.push_back(dis[j] * -1); + } + + auto search_result_on_raw_index = (QueryResult*)c_search_result_on_smallIndex; + search_result_on_raw_index->internal_seg_offsets_ = vec_ids; + search_result_on_raw_index->result_distances_ = vec_dis; + + auto binary_set = indexing->Serialize(conf); + void* c_load_index_info = nullptr; + status = NewLoadIndexInfo(&c_load_index_info); + assert(status.error_code == Success); + std::string index_type_key = "index_type"; + std::string index_type_value = "IVF_PQ"; + std::string index_mode_key = "index_mode"; + std::string index_mode_value = "cpu"; + std::string metric_type_key = "metric_type"; + std::string metric_type_value = "L2"; + + AppendIndexParam(c_load_index_info, index_type_key.c_str(), index_type_value.c_str()); + AppendIndexParam(c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str()); + AppendIndexParam(c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str()); + AppendFieldInfo(c_load_index_info, "fakevec", 0); + AppendIndex(c_load_index_info, (CBinarySet)&binary_set); + + status = UpdateSegmentIndex(segment, c_load_index_info); + assert(status.error_code == Success); + + CQueryResult c_search_result_on_bigIndex; + auto res_after_load_index = Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_bigIndex); + assert(res_after_load_index.error_code == Success); + + auto search_result_on_raw_index_json = QueryResultToJson(*search_result_on_raw_index); + auto search_result_on_bigIndex_json = QueryResultToJson((*(QueryResult*)c_search_result_on_bigIndex)); + std::cout << search_result_on_raw_index_json.dump(1) << std::endl; + std::cout << search_result_on_bigIndex_json.dump(1) << std::endl; + + ASSERT_EQ(search_result_on_raw_index_json.dump(1), search_result_on_bigIndex_json.dump(1)); + + DeleteLoadIndexInfo(c_load_index_info); + DeletePlan(plan); + DeletePlaceholderGroup(placeholderGroup); + DeleteQueryResult(c_search_result_on_smallIndex); + DeleteQueryResult(c_search_result_on_bigIndex); + DeleteCollection(collection); + DeleteSegment(segment); +} + +TEST(CApiTest, UpdateSegmentIndex_With_Predicate_Range) { + // insert data to segment + constexpr auto DIM = 16; + constexpr auto K = 5; + + std::string schema_string = generate_collection_shema("L2", "16"); + auto collection = NewCollection(schema_string.c_str()); + auto schema = ((segcore::Collection*)collection)->get_schema(); + auto segment = NewSegment(collection, 0); + + auto N = 1000 * 1000; + auto dataset = DataGen(schema, N); + auto vec_col = dataset.get_col(0); + auto query_ptr = vec_col.data() + 420000 * DIM; + + PreInsert(segment, N); + auto ins_res = Insert(segment, 0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data, + dataset.raw_.sizeof_per_row, dataset.raw_.count); + assert(ins_res.error_code == Success); + + const char* dsl_string = R"({ + "bool": { + "must": [ + { + "range": { + "counter": { + "GE": 420000, + "LT": 420010 + } + } + }, + { + "vector": { + "fakevec": { + "metric_type": "L2", + "params": { + "nprobe": 10 + }, + "query": "$0", + "topk": 5 + } + } + } + ] + } + })"; + + // create place_holder_group + int num_queries = 10; + auto raw_group = CreatePlaceholderGroupFromBlob(num_queries, DIM, query_ptr); + auto blob = raw_group.SerializeAsString(); + + // search on segment's small index + void* plan = nullptr; + auto status = CreatePlan(collection, dsl_string, &plan); + assert(status.error_code == Success); + + void* placeholderGroup = nullptr; + status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup); + assert(status.error_code == Success); + + std::vector placeholderGroups; + placeholderGroups.push_back(placeholderGroup); + Timestamp time = 10000000; + + CQueryResult c_search_result_on_smallIndex; + auto res_before_load_index = + Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_smallIndex); + assert(res_before_load_index.error_code == Success); + + // load index to segment + auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, + {milvus::knowhere::meta::TOPK, K}, + {milvus::knowhere::IndexParams::nlist, 100}, + {milvus::knowhere::IndexParams::nprobe, 10}, + {milvus::knowhere::IndexParams::m, 4}, + {milvus::knowhere::IndexParams::nbits, 8}, + {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {milvus::knowhere::meta::DEVICEID, 0}}; + + auto indexing = generate_index(vec_col.data(), conf, DIM, K, N, IndexEnum::INDEX_FAISS_IVFPQ); + + // gen query dataset + auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto result_on_index = indexing->Query(query_dataset, conf, nullptr); + auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); + auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + std::vector vec_ids(ids, ids + K * num_queries); + std::vector vec_dis; + for (int j = 0; j < K * num_queries; ++j) { + vec_dis.push_back(dis[j] * -1); + } + + auto search_result_on_raw_index = (QueryResult*)c_search_result_on_smallIndex; + search_result_on_raw_index->internal_seg_offsets_ = vec_ids; + search_result_on_raw_index->result_distances_ = vec_dis; + + auto binary_set = indexing->Serialize(conf); + void* c_load_index_info = nullptr; + status = NewLoadIndexInfo(&c_load_index_info); + assert(status.error_code == Success); + std::string index_type_key = "index_type"; + std::string index_type_value = "IVF_PQ"; + std::string index_mode_key = "index_mode"; + std::string index_mode_value = "cpu"; + std::string metric_type_key = "metric_type"; + std::string metric_type_value = "L2"; + + AppendIndexParam(c_load_index_info, index_type_key.c_str(), index_type_value.c_str()); + AppendIndexParam(c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str()); + AppendIndexParam(c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str()); + AppendFieldInfo(c_load_index_info, "fakevec", 0); + AppendIndex(c_load_index_info, (CBinarySet)&binary_set); + + status = UpdateSegmentIndex(segment, c_load_index_info); + assert(status.error_code == Success); + + CQueryResult c_search_result_on_bigIndex; + auto res_after_load_index = Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_bigIndex); + assert(res_after_load_index.error_code == Success); + + auto search_result_on_bigIndex = (*(QueryResult*)c_search_result_on_bigIndex); + for (int i = 0; i < num_queries; ++i) { + auto offset = i * K; + ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i); + ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset], + search_result_on_raw_index->result_distances_[offset]); + } + + DeleteLoadIndexInfo(c_load_index_info); + DeletePlan(plan); + DeletePlaceholderGroup(placeholderGroup); + DeleteQueryResult(c_search_result_on_smallIndex); + DeleteQueryResult(c_search_result_on_bigIndex); + DeleteCollection(collection); + DeleteSegment(segment); +} + +TEST(CApiTest, UpdateSegmentIndex_With_Predicate_Term) { + // insert data to segment + constexpr auto DIM = 16; + constexpr auto K = 5; + + std::string schema_string = generate_collection_shema("L2", "16"); + auto collection = NewCollection(schema_string.c_str()); + auto schema = ((segcore::Collection*)collection)->get_schema(); + auto segment = NewSegment(collection, 0); + + auto N = 1000 * 1000; + auto dataset = DataGen(schema, N); + auto vec_col = dataset.get_col(0); + auto query_ptr = vec_col.data() + 420000 * DIM; + + PreInsert(segment, N); + auto ins_res = Insert(segment, 0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_.raw_data, + dataset.raw_.sizeof_per_row, dataset.raw_.count); + assert(ins_res.error_code == Success); + + const char* dsl_string = R"({ + "bool": { + "must": [ + { + "term": { + "counter": { + "values": [420000, 420001, 420002, 420003, 420004] + } + } + }, + { + "vector": { + "fakevec": { + "metric_type": "L2", + "params": { + "nprobe": 10 + }, + "query": "$0", + "topk": 5 + } + } + } + ] + } + })"; + + // create place_holder_group + int num_queries = 5; + auto raw_group = CreatePlaceholderGroupFromBlob(num_queries, DIM, query_ptr); + auto blob = raw_group.SerializeAsString(); + + // search on segment's small index + void* plan = nullptr; + auto status = CreatePlan(collection, dsl_string, &plan); + assert(status.error_code == Success); + + void* placeholderGroup = nullptr; + status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup); + assert(status.error_code == Success); + + std::vector placeholderGroups; + placeholderGroups.push_back(placeholderGroup); + Timestamp time = 10000000; + + CQueryResult c_search_result_on_smallIndex; + auto res_before_load_index = + Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_smallIndex); + assert(res_before_load_index.error_code == Success); + + // load index to segment + auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, + {milvus::knowhere::meta::TOPK, K}, + {milvus::knowhere::IndexParams::nlist, 100}, + {milvus::knowhere::IndexParams::nprobe, 10}, + {milvus::knowhere::IndexParams::m, 4}, + {milvus::knowhere::IndexParams::nbits, 8}, + {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {milvus::knowhere::meta::DEVICEID, 0}}; + + auto indexing = generate_index(vec_col.data(), conf, DIM, K, N, IndexEnum::INDEX_FAISS_IVFPQ); + + // gen query dataset + auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto result_on_index = indexing->Query(query_dataset, conf, nullptr); + auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); + auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + std::vector vec_ids(ids, ids + K * num_queries); + std::vector vec_dis; + for (int j = 0; j < K * num_queries; ++j) { + vec_dis.push_back(dis[j] * -1); + } + + auto search_result_on_raw_index = (QueryResult*)c_search_result_on_smallIndex; + search_result_on_raw_index->internal_seg_offsets_ = vec_ids; + search_result_on_raw_index->result_distances_ = vec_dis; + + auto binary_set = indexing->Serialize(conf); + void* c_load_index_info = nullptr; + status = NewLoadIndexInfo(&c_load_index_info); + assert(status.error_code == Success); + std::string index_type_key = "index_type"; + std::string index_type_value = "IVF_PQ"; + std::string index_mode_key = "index_mode"; + std::string index_mode_value = "cpu"; + std::string metric_type_key = "metric_type"; + std::string metric_type_value = "L2"; + + AppendIndexParam(c_load_index_info, index_type_key.c_str(), index_type_value.c_str()); + AppendIndexParam(c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str()); + AppendIndexParam(c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str()); + AppendFieldInfo(c_load_index_info, "fakevec", 0); + AppendIndex(c_load_index_info, (CBinarySet)&binary_set); + + status = UpdateSegmentIndex(segment, c_load_index_info); + assert(status.error_code == Success); + + CQueryResult c_search_result_on_bigIndex; + auto res_after_load_index = Search(segment, plan, placeholderGroups.data(), &time, 1, &c_search_result_on_bigIndex); + assert(res_after_load_index.error_code == Success); + + auto search_result_on_bigIndex = (*(QueryResult*)c_search_result_on_bigIndex); + for (int i = 0; i < num_queries; ++i) { + auto offset = i * K; + ASSERT_EQ(search_result_on_bigIndex.internal_seg_offsets_[offset], 420000 + i); + ASSERT_EQ(search_result_on_bigIndex.result_distances_[offset], + search_result_on_raw_index->result_distances_[offset]); + } + + DeleteLoadIndexInfo(c_load_index_info); + DeletePlan(plan); + DeletePlaceholderGroup(placeholderGroup); + DeleteQueryResult(c_search_result_on_smallIndex); + DeleteQueryResult(c_search_result_on_bigIndex); + DeleteCollection(collection); + DeleteSegment(segment); } \ No newline at end of file diff --git a/internal/querynode/load_index_service_test.go b/internal/querynode/load_index_service_test.go index 564719be81..4145ba3bcd 100644 --- a/internal/querynode/load_index_service_test.go +++ b/internal/querynode/load_index_service_test.go @@ -16,23 +16,6 @@ import ( "github.com/zilliztech/milvus-distributed/internal/querynode/client" ) -//func TestLoadIndexClient_LoadIndex(t *testing.T) { -// pulsarURL := Params.PulsarAddress -// loadIndexChannels := Params.LoadIndexChannelNames -// loadIndexClient := client.NewLoadIndexClient(context.Background(), pulsarURL, loadIndexChannels) -// -// loadIndexPath := "collection0-segment0-field0" -// loadIndexPaths := make([]string, 0) -// loadIndexPaths = append(loadIndexPaths, loadIndexPath) -// -// indexParams := make(map[string]string) -// indexParams["index_type"] = "IVF_PQ" -// indexParams["index_mode"] = "cpu" -// -// loadIndexClient.LoadIndex(loadIndexPaths, 0, 0, "field0", indexParams) -// loadIndexClient.Close() -//} - func TestLoadIndexService(t *testing.T) { node := newQueryNode() collectionID := rand.Int63n(1000000) diff --git a/scripts/gofmt.sh b/scripts/gofmt.sh index b1648ab1c7..24a974477f 100755 --- a/scripts/gofmt.sh +++ b/scripts/gofmt.sh @@ -14,7 +14,7 @@ files=() files_need_gofmt=() if [ -f "$1" ];then - files+=("$1") + files+=("$1") fi if [ -d "$1" ];then diff --git a/tools/check/go.mod b/tools/check/go.mod new file mode 100644 index 0000000000..7d15e9b900 --- /dev/null +++ b/tools/check/go.mod @@ -0,0 +1,14 @@ +module github.com/zilliztech/milvus-distributed/tools/check + +go 1.15 + +require ( + github.com/fatih/color v1.10.0 // indirect + github.com/git-hooks/git-hooks v1.3.1 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mgechev/revive v1.0.2 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/urfave/cli v1.22.5 // indirect + golang.org/x/sys v0.0.0-20201223074533-0d417f636930 // indirect + golang.org/x/tools v0.0.0-20201230224404-63754364767c // indirect +) diff --git a/tools/check/go.sum b/tools/check/go.sum new file mode 100644 index 0000000000..0f33a62787 --- /dev/null +++ b/tools/check/go.sum @@ -0,0 +1,93 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab h1:1WOH7EEbhb6OZWcIU5RpQx5rmHm1xEUda8Qiw4UzNlU= +github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab/go.mod h1:5MSsYMW59C/HfIUsthTRDxRoMQctcmAVb1JnNSQXERA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/git-hooks/git-hooks v1.3.1 h1:ypdvNTXMiITXQxuqIl6t1f8R3V1FrUXPi1CwQ4guClo= +github.com/git-hooks/git-hooks v1.3.1/go.mod h1:RBqjgxUpRSLI4AxVZGV1FsODclMjpsJIaDEyJSoK3ws= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= +github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= +github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg= +github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/testify v0.0.0-20141015234014-d6577e08ec30/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200225230052-807dcd883420 h1:4RJNOV+2rLxMEfr6QIpC7GEv9MjD6ApGXTCLrNF9+eA= +golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201230224404-63754364767c h1:xx3+TTG3yS1I6Ola5Kapxr5vZu85vKkcwKyV6ke9fHA= +golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= diff --git a/tools/check/revive.toml b/tools/check/revive.toml new file mode 100644 index 0000000000..04df7e2965 --- /dev/null +++ b/tools/check/revive.toml @@ -0,0 +1,30 @@ +ignoreGeneratedHeader = false +severity = "error" +confidence = 0.8 +errorCode = 1 +warningCode = 0 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +[rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.package-comments] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] +[rule.empty-block] +[rule.superfluous-else] +[rule.unused-parameter] +[rule.unreachable-code] +[rule.redefines-builtin-id]