fix clang tidy warnings (#3362)

* fix codacy warning

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

* update codacy rules

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

* update index unittest build option

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
pull/3370/head^2
Cai Yudong 2020-08-20 18:34:44 +08:00 committed by GitHub
parent b437639b7a
commit fb70337c74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 142 additions and 137 deletions

View File

@ -1,12 +1,12 @@
exclude_paths: exclude_paths:
- 'core/src/index/thirdparty/**'
- 'sdk/build-support/**'
- 'sdk/thirdparty/**'
- 'core/build-support/**' - 'core/build-support/**'
- 'core/src/index/thirdparty/**'
- 'core/src/grpc/gen-milvus/**' - 'core/src/grpc/gen-milvus/**'
- 'core/src/grpc/gen-status/**' - 'core/src/grpc/gen-status/**'
- 'sdk/grpc-gen/**'
- 'core/thirdparty/**' - 'core/thirdparty/**'
- 'sdk/build-support/**'
- 'sdk/grpc-gen/**'
- 'sdk/thirdparty/**'
- 'ci/**' - 'ci/**'
- 'docker/**' - 'docker/**'
- 'docs/**' - 'docs/**'

View File

@ -163,7 +163,7 @@ StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const
auto meta = iter.first.c_str(); auto meta = iter.first.c_str();
size_t meta_length = iter.first.length(); size_t meta_length = iter.first.length();
fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length)); fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length); fs_ptr->writer_ptr_->Write(meta, meta_length);
auto binary = iter.second; auto binary = iter.second;
int64_t binary_length = binary->size; int64_t binary_length = binary->size;

View File

@ -194,7 +194,7 @@ VectorIndexFormat::WriteIndex(const storage::FSHandlerPtr& fs_ptr, const std::st
auto meta = iter.first.c_str(); auto meta = iter.first.c_str();
size_t meta_length = iter.first.length(); size_t meta_length = iter.first.length();
fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length)); fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length); fs_ptr->writer_ptr_->Write(meta, meta_length);
auto binary = iter.second; auto binary = iter.second;
int64_t binary_length = binary->size; int64_t binary_length = binary->size;

View File

@ -223,7 +223,7 @@ ConfigMgr::Set(const std::string& name, const std::string& value, bool update) {
throw ConfigStatus(SetReturn::IMMUTABLE, "Config " + name + " is not modifiable"); throw ConfigStatus(SetReturn::IMMUTABLE, "Config " + name + " is not modifiable");
} }
} catch (ConfigStatus& cs) { } catch (ConfigStatus& cs) {
throw cs; throw;
} catch (...) { } catch (...) {
throw "Config " + name + " not found."; throw "Config " + name + " not found.";
} }

View File

@ -12,7 +12,6 @@
#include "db/IDGenerator.h" #include "db/IDGenerator.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <assert.h>
#include <fiu/fiu-local.h> #include <fiu/fiu-local.h>
#include <chrono> #include <chrono>
#include <iostream> #include <iostream>

View File

@ -153,7 +153,7 @@ IsVectorField(const engine::snapshot::FieldPtr& field) {
return false; return false;
} }
engine::DataType ftype = static_cast<engine::DataType>(field->GetFtype()); auto ftype = static_cast<engine::DataType>(field->GetFtype());
return IsVectorField(ftype); return IsVectorField(ftype);
} }

View File

@ -11,16 +11,8 @@
#include "db/insert/MemManagerFactory.h" #include "db/insert/MemManagerFactory.h"
#include "MemManagerImpl.h" #include "MemManagerImpl.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <time.h>
#include <cstdlib>
#include <memory> #include <memory>
#include <regex>
#include <sstream>
#include <string>
namespace milvus { namespace milvus {
namespace engine { namespace engine {

View File

@ -13,8 +13,6 @@
#include "MemManager.h" #include "MemManager.h"
#include <memory>
namespace milvus { namespace milvus {
namespace engine { namespace engine {

View File

@ -24,8 +24,7 @@ namespace milvus::engine::meta {
class SqliteEngine : public MetaEngine { class SqliteEngine : public MetaEngine {
public: public:
explicit SqliteEngine(const DBMetaOptions& options); explicit SqliteEngine(const DBMetaOptions& options);
virtual ~SqliteEngine();
~SqliteEngine();
public: public:
Status Status

View File

@ -122,7 +122,8 @@ class Operations : public std::enable_shared_from_this<Operations> {
virtual Status virtual Status
operator()(StorePtr store); operator()(StorePtr store);
virtual Status
Status
Push(bool sync = true); Push(bool sync = true);
virtual Status virtual Status

View File

@ -114,8 +114,9 @@ PartitionCommitOperation::DoExecute(StorePtr store) {
row_cnt = resource_->GetRowCount(); row_cnt = resource_->GetRowCount();
size = resource_->GetSize(); size = resource_->GetSize();
auto erase_sc = [&](SegmentCommitPtr& sc) { auto erase_sc = [&](SegmentCommitPtr& sc) {
if (!sc) if (!sc) {
return; return;
}
auto prev_sc = GetStartedSS()->GetSegmentCommitBySegmentId(sc->GetSegmentId()); auto prev_sc = GetStartedSS()->GetSegmentCommitBySegmentId(sc->GetSegmentId());
if (prev_sc) { if (prev_sc) {
resource_->GetMappings().erase(prev_sc->GetID()); resource_->GetMappings().erase(prev_sc->GetID());

View File

@ -100,7 +100,7 @@ Segment::ToString() const {
ss << "id=" << GetID() << ", "; ss << "id=" << GetID() << ", ";
ss << "partition_id=" << GetPartitionId() << ", "; ss << "partition_id=" << GetPartitionId() << ", ";
ss << "collection_id=" << GetCollectionId() << ", "; ss << "collection_id=" << GetCollectionId() << ", ";
ss << "num=" << (NUM_TYPE)GetNum() << ", "; ss << "num=" << GetNum() << ", ";
ss << "state=" << GetState() << ", "; ss << "state=" << GetState() << ", ";
return ss.str(); return ss.str();
} }

View File

@ -37,22 +37,17 @@ SnapshotHolder::~SnapshotHolder() {
Status Status
SnapshotHolder::Load(StorePtr store, ScopedSnapshotT& ss, ID_TYPE id, bool scoped) { SnapshotHolder::Load(StorePtr store, ScopedSnapshotT& ss, ID_TYPE id, bool scoped) {
Status status;
if (id > max_id_) { if (id > max_id_) {
CollectionCommitPtr cc; CollectionCommitPtr cc;
status = LoadNoLock(id, cc, store); STATUS_CHECK(LoadNoLock(id, cc, store));
if (!status.ok()) STATUS_CHECK(Add(store, id));
return status;
status = Add(store, id);
if (!status.ok())
return status;
} }
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
if (id == 0 || id == max_id_) { if (id == 0 || id == max_id_) {
auto raw = active_.at(max_id_); auto raw = active_.at(max_id_);
ss = ScopedSnapshotT(raw, scoped); ss = ScopedSnapshotT(raw, scoped);
return status; return Status::OK();
} }
if (id < min_id_) { if (id < min_id_) {
std::stringstream emsg; std::stringstream emsg;
@ -71,7 +66,7 @@ SnapshotHolder::Load(StorePtr store, ScopedSnapshotT& ss, ID_TYPE id, bool scope
return Status(SS_NOT_FOUND_ERROR, emsg.str()); return Status(SS_NOT_FOUND_ERROR, emsg.str());
} }
ss = ScopedSnapshotT(it->second, scoped); ss = ScopedSnapshotT(it->second, scoped);
return status; return Status::OK();
} }
Status Status

View File

@ -25,9 +25,7 @@ namespace snapshot {
Status Status
Snapshots::DropCollection(ID_TYPE collection_id, const LSN_TYPE& lsn) { Snapshots::DropCollection(ID_TYPE collection_id, const LSN_TYPE& lsn) {
ScopedSnapshotT ss; ScopedSnapshotT ss;
auto status = GetSnapshot(ss, collection_id); STATUS_CHECK(GetSnapshot(ss, collection_id));
if (!status.ok())
return status;
return DoDropCollection(ss, lsn); return DoDropCollection(ss, lsn);
} }
@ -189,25 +187,21 @@ Snapshots::GetHolder(const ID_TYPE& collection_id, SnapshotHolderPtr& holder) co
Status Status
Snapshots::LoadHolder(StorePtr store, const ID_TYPE& collection_id, SnapshotHolderPtr& holder) { Snapshots::LoadHolder(StorePtr store, const ID_TYPE& collection_id, SnapshotHolderPtr& holder) {
Status status;
{ {
std::shared_lock<std::shared_timed_mutex> lock(mutex_); std::shared_lock<std::shared_timed_mutex> lock(mutex_);
status = GetHolderNoLock(collection_id, holder); auto status = GetHolderNoLock(collection_id, holder);
if (status.ok() && holder) if (status.ok() && holder) {
return status; return status;
}
} }
status = LoadNoLock(store, collection_id, holder); STATUS_CHECK(LoadNoLock(store, collection_id, holder));
if (!status.ok())
return status;
std::unique_lock<std::shared_timed_mutex> lock(mutex_); std::unique_lock<std::shared_timed_mutex> lock(mutex_);
holders_[collection_id] = holder; holders_[collection_id] = holder;
ScopedSnapshotT ss; ScopedSnapshotT ss;
status = holder->Load(store, ss); STATUS_CHECK(holder->Load(store, ss));
if (!status.ok())
return status;
name_id_map_[ss->GetName()] = collection_id; name_id_map_[ss->GetName()] = collection_id;
return status; return Status::OK();
} }
Status Status

View File

@ -18,9 +18,6 @@
namespace milvus { namespace milvus {
namespace engine { namespace engine {
WalFile::WalFile() {
}
WalFile::~WalFile() { WalFile::~WalFile() {
CloseFile(); CloseFile();
} }

View File

@ -24,8 +24,8 @@ namespace engine {
class WalFile { class WalFile {
public: public:
WalFile(); WalFile() = default;
~WalFile(); virtual ~WalFile();
bool bool
IsOpened() const { IsOpened() const {
@ -60,7 +60,7 @@ class WalFile {
} }
inline int64_t inline int64_t
Write(void* data, int64_t length) { Write(const void* data, int64_t length) {
if (file_ == nullptr) { if (file_ == nullptr) {
return 0; return 0;
} }

View File

@ -11,6 +11,7 @@
#pragma once #pragma once
#include "config/ServerConfig.h"
#include "db/DB.h" #include "db/DB.h"
#include "db/IDGenerator.h" #include "db/IDGenerator.h"
#include "db/Types.h" #include "db/Types.h"

View File

@ -58,7 +58,7 @@ WalOperationCodec::WriteInsertOperation(const WalFilePtr& file, const std::strin
int32_t part_name_length = partition_name.size(); int32_t part_name_length = partition_name.size();
total_bytes += file->Write<int32_t>(&part_name_length); total_bytes += file->Write<int32_t>(&part_name_length);
if (part_name_length > 0) { if (part_name_length > 0) {
total_bytes += file->Write((void*)partition_name.data(), part_name_length); total_bytes += file->Write(partition_name.data(), part_name_length);
} }
// write fixed data // write fixed data
@ -71,11 +71,11 @@ WalOperationCodec::WriteInsertOperation(const WalFilePtr& file, const std::strin
int32_t field_name_length = pair.first.size(); int32_t field_name_length = pair.first.size();
total_bytes += file->Write<int32_t>(&field_name_length); total_bytes += file->Write<int32_t>(&field_name_length);
total_bytes += file->Write((void*)pair.first.data(), field_name_length); total_bytes += file->Write(pair.first.data(), field_name_length);
int64_t data_size = pair.second->data_.size(); int64_t data_size = pair.second->data_.size();
total_bytes += file->Write<int64_t>(&data_size); total_bytes += file->Write<int64_t>(&data_size);
total_bytes += file->Write((void*)pair.second->data_.data(), data_size); total_bytes += file->Write(pair.second->data_.data(), data_size);
} }
// TODO: write variable data // TODO: write variable data
@ -132,7 +132,7 @@ WalOperationCodec::WriteDeleteOperation(const WalFilePtr& file, const IDNumbers&
int64_t id_count = entity_ids.size(); int64_t id_count = entity_ids.size();
total_bytes += file->Write<int64_t>(&id_count); total_bytes += file->Write<int64_t>(&id_count);
total_bytes += file->Write((void*)entity_ids.data(), id_count * sizeof(idx_t)); total_bytes += file->Write(entity_ids.data(), id_count * sizeof(idx_t));
// write operation id again // write operation id again
// Note: makesure operation id is written at end, so that wal cleanup thread know which file can be deleted // Note: makesure operation id is written at end, so that wal cleanup thread know which file can be deleted

View File

@ -63,8 +63,9 @@ KnowhereResource::Initialize() {
#ifdef MILVUS_GPU_VERSION #ifdef MILVUS_GPU_VERSION
bool enable_gpu = config.gpu.enable(); bool enable_gpu = config.gpu.enable();
fiu_do_on("KnowhereResource.Initialize.disable_gpu", enable_gpu = false); fiu_do_on("KnowhereResource.Initialize.disable_gpu", enable_gpu = false);
if (not enable_gpu) if (!enable_gpu) {
return Status::OK(); return Status::OK();
}
struct GpuResourceSetting { struct GpuResourceSetting {
int64_t pinned_memory = 256 * M_BYTE; int64_t pinned_memory = 256 * M_BYTE;

View File

@ -19,8 +19,9 @@ namespace knowhere {
ConfAdapterPtr ConfAdapterPtr
AdapterMgr::GetAdapter(const IndexType type) { AdapterMgr::GetAdapter(const IndexType type) {
if (!init_) if (!init_) {
RegisterAdapter(); RegisterAdapter();
}
try { try {
return collection_.at(type)(); return collection_.at(type)();

View File

@ -28,21 +28,21 @@ MemoryIOWriter::operator()(const void* ptr, size_t size, size_t nitems) {
total = total_need * magic_num; total = total_need * magic_num;
rp = size * nitems; rp = size * nitems;
data_ = new uint8_t[total]; data_ = new uint8_t[total];
memcpy((void*)(data_), ptr, rp); memcpy(data_, ptr, rp);
return nitems; return nitems;
} }
if (total_need > total) { if (total_need > total) {
total = total_need * magic_num; total = total_need * magic_num;
auto new_data = new uint8_t[total]; auto new_data = new uint8_t[total];
memcpy((void*)new_data, (void*)data_, rp); memcpy(new_data, data_, rp);
delete[] data_; delete[] data_;
data_ = new_data; data_ = new_data;
memcpy((void*)(data_ + rp), ptr, size * nitems); memcpy((data_ + rp), ptr, size * nitems);
rp = total_need; rp = total_need;
} else { } else {
memcpy((void*)(data_ + rp), ptr, size * nitems); memcpy((data_ + rp), ptr, size * nitems);
rp = total_need; rp = total_need;
} }
@ -51,12 +51,14 @@ MemoryIOWriter::operator()(const void* ptr, size_t size, size_t nitems) {
size_t size_t
MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) { MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) {
if (rp >= total) if (rp >= total) {
return 0; return 0;
}
size_t nremain = (total - rp) / size; size_t nremain = (total - rp) / size;
if (nremain < nitems) if (nremain < nitems) {
nitems = nremain; nitems = nremain;
memcpy(ptr, (void*)(data_ + rp), size * nitems); }
memcpy(ptr, (data_ + rp), size * nitems);
rp += size * nitems; rp += size * nitems;
return nitems; return nitems;
} }

View File

@ -52,7 +52,7 @@ NsgIndex::Build_with_ids(size_t nb, float* data, const int64_t* ids, const Build
// ori_data_ = new float[ntotal * dimension]; // ori_data_ = new float[ntotal * dimension];
ids_ = new int64_t[ntotal]; ids_ = new int64_t[ntotal];
// memcpy((void*)ori_data_, (void*)data, sizeof(float) * ntotal * dimension); // memcpy((void*)ori_data_, (void*)data, sizeof(float) * ntotal * dimension);
memcpy((void*)ids_, (void*)ids, sizeof(int64_t) * ntotal); memcpy(ids_, ids, sizeof(int64_t) * ntotal);
search_length = parameters.search_length; search_length = parameters.search_length;
out_degree = parameters.out_degree; out_degree = parameters.out_degree;
@ -153,8 +153,9 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
} }
while (count < buffer_size) { while (count < buffer_size) {
node_t id = rand_r(&seed) % ntotal; node_t id = rand_r(&seed) % ntotal;
if (has_calculated_dist[id]) if (has_calculated_dist[id]) {
continue; // duplicate id continue; // duplicate id
}
// init_ids.push_back(id); // init_ids.push_back(id);
init_ids[count] = id; init_ids[count] = id;
++count; ++count;
@ -193,26 +194,29 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
node_t start_pos = resset[cursor].id; node_t start_pos = resset[cursor].id;
auto& wait_for_search_node_vec = graph[start_pos]; auto& wait_for_search_node_vec = graph[start_pos];
for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { for (node_t id : wait_for_search_node_vec) {
node_t id = wait_for_search_node_vec[i]; if (has_calculated_dist[id]) {
if (has_calculated_dist[id])
continue; continue;
}
has_calculated_dist[id] = true; has_calculated_dist[id] = true;
float dist = distance_->Compare(query, data + dimension * id, dimension); float dist = distance_->Compare(query, data + dimension * id, dimension);
Neighbor nn(id, dist, false); Neighbor nn(id, dist, false);
fullset.push_back(nn); fullset.push_back(nn);
if (dist >= resset[buffer_size - 1].distance) if (dist >= resset[buffer_size - 1].distance) {
continue; continue;
}
size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node
if (pos < nearest_updated_pos) if (pos < nearest_updated_pos) {
nearest_updated_pos = pos; nearest_updated_pos = pos;
}
// assert(buffer_size + 1 >= resset.size()); // assert(buffer_size + 1 >= resset.size());
if (buffer_size + 1 < resset.size()) if (buffer_size + 1 < resset.size()) {
++buffer_size; ++buffer_size;
}
} }
} }
if (cursor >= nearest_updated_pos) { if (cursor >= nearest_updated_pos) {
@ -255,8 +259,9 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
} }
while (count < buffer_size) { while (count < buffer_size) {
node_t id = rand_r(&seed) % ntotal; node_t id = rand_r(&seed) % ntotal;
if (has_calculated_dist[id]) if (has_calculated_dist[id]) {
continue; // duplicate id continue; // duplicate id
}
// init_ids.push_back(id); // init_ids.push_back(id);
init_ids[count] = id; init_ids[count] = id;
++count; ++count;
@ -291,26 +296,29 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
node_t start_pos = resset[cursor].id; node_t start_pos = resset[cursor].id;
auto& wait_for_search_node_vec = graph[start_pos]; auto& wait_for_search_node_vec = graph[start_pos];
for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { for (node_t id : wait_for_search_node_vec) {
node_t id = wait_for_search_node_vec[i]; if (has_calculated_dist[id]) {
if (has_calculated_dist[id])
continue; continue;
}
has_calculated_dist[id] = true; has_calculated_dist[id] = true;
float dist = distance_->Compare(data + dimension * id, query, dimension); float dist = distance_->Compare(data + dimension * id, query, dimension);
Neighbor nn(id, dist, false); Neighbor nn(id, dist, false);
fullset.push_back(nn); fullset.push_back(nn);
if (dist >= resset[buffer_size - 1].distance) if (dist >= resset[buffer_size - 1].distance) {
continue; continue;
}
size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node
if (pos < nearest_updated_pos) if (pos < nearest_updated_pos) {
nearest_updated_pos = pos; nearest_updated_pos = pos;
}
// assert(buffer_size + 1 >= resset.size()); // assert(buffer_size + 1 >= resset.size());
if (buffer_size + 1 < resset.size()) if (buffer_size + 1 < resset.size()) {
++buffer_size; // trick ++buffer_size; // trick
}
} }
} }
if (cursor >= nearest_updated_pos) { if (cursor >= nearest_updated_pos) {
@ -349,8 +357,9 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
} }
while (count < buffer_size) { while (count < buffer_size) {
node_t id = rand_r(&seed) % ntotal; node_t id = rand_r(&seed) % ntotal;
if (has_calculated_dist[id]) if (has_calculated_dist[id]) {
continue; // duplicate id continue; // duplicate id
}
init_ids[count] = id; init_ids[count] = id;
++count; ++count;
has_calculated_dist[id] = true; has_calculated_dist[id] = true;
@ -383,24 +392,26 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
node_t start_pos = resset[cursor].id; node_t start_pos = resset[cursor].id;
auto& wait_for_search_node_vec = graph[start_pos]; auto& wait_for_search_node_vec = graph[start_pos];
for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { for (node_t id : wait_for_search_node_vec) {
node_t id = wait_for_search_node_vec[i]; if (has_calculated_dist[id]) {
if (has_calculated_dist[id])
continue; continue;
}
has_calculated_dist[id] = true; has_calculated_dist[id] = true;
float dist = distance_->Compare(query, data + dimension * id, dimension); float dist = distance_->Compare(query, data + dimension * id, dimension);
if (dist >= resset[buffer_size - 1].distance) if (dist >= resset[buffer_size - 1].distance) {
continue; continue;
}
//// difference from other GetNeighbors //// difference from other GetNeighbors
Neighbor nn(id, dist, false); Neighbor nn(id, dist, false);
/////////////////////////////////////// ///////////////////////////////////////
size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node
if (pos < nearest_updated_pos) if (pos < nearest_updated_pos) {
nearest_updated_pos = pos; nearest_updated_pos = pos;
}
//>> Debug code //>> Debug code
///// /////
@ -408,8 +419,9 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
// nearest_updated_pos << std::endl; // nearest_updated_pos << std::endl;
///// /////
// trick: avoid search query search_length < init_ids.size() ... // trick: avoid search query search_length < init_ids.size() ...
if (buffer_size + 1 < resset.size()) if (buffer_size + 1 < resset.size()) {
++buffer_size; ++buffer_size;
}
} }
} }
if (cursor >= nearest_updated_pos) { if (cursor >= nearest_updated_pos) {
@ -423,7 +435,7 @@ NsgIndex::GetNeighbors(const float* query, float* data, std::vector<Neighbor>& r
void void
NsgIndex::Link(float* data) { NsgIndex::Link(float* data) {
float* cut_graph_dist = new float[ntotal * out_degree]; auto cut_graph_dist = new float[ntotal * out_degree];
nsg.resize(ntotal); nsg.resize(ntotal);
#pragma omp parallel #pragma omp parallel
@ -475,8 +487,9 @@ NsgIndex::SyncPrune(float* data, size_t n, std::vector<Neighbor>& pool, boost::d
// avoid lose nearest neighbor in knng // avoid lose nearest neighbor in knng
for (size_t i = 0; i < knng[n].size(); ++i) { for (size_t i = 0; i < knng[n].size(); ++i) {
auto id = knng[n][i]; auto id = knng[n][i];
if (has_calculated[id]) if (has_calculated[id]) {
continue; continue;
}
float dist = distance_->Compare(data + dimension * n, data + dimension * id, dimension); float dist = distance_->Compare(data + dimension * n, data + dimension * id, dimension);
pool.emplace_back(Neighbor(id, dist, true)); pool.emplace_back(Neighbor(id, dist, true));
} }
@ -513,8 +526,9 @@ NsgIndex::InterInsert(float* data, unsigned n, std::vector<std::mutex>& mutex_ve
auto& neighbor_id_pool = nsg[current]; auto& neighbor_id_pool = nsg[current];
float* neighbor_dist_pool = cut_graph_dist + current * out_degree; float* neighbor_dist_pool = cut_graph_dist + current * out_degree;
for (size_t i = 0; i < out_degree; ++i) { for (size_t i = 0; i < out_degree; ++i) {
if (neighbor_dist_pool[i] == -1) if (neighbor_dist_pool[i] == -1) {
break; break;
}
size_t current_neighbor = neighbor_id_pool[i]; // center's neighbor id size_t current_neighbor = neighbor_id_pool[i]; // center's neighbor id
auto& nsn_id_pool = nsg[current_neighbor]; // nsn => neighbor's neighbor auto& nsn_id_pool = nsg[current_neighbor]; // nsn => neighbor's neighbor
@ -525,8 +539,9 @@ NsgIndex::InterInsert(float* data, unsigned n, std::vector<std::mutex>& mutex_ve
{ {
LockGuard lk(mutex_vec[current_neighbor]); LockGuard lk(mutex_vec[current_neighbor]);
for (size_t j = 0; j < out_degree; ++j) { for (size_t j = 0; j < out_degree; ++j) {
if (nsn_dist_pool[j] == -1) if (nsn_dist_pool[j] == -1) {
break; break;
}
// At least one edge can be connected back // At least one edge can be connected back
if (n == nsn_id_pool[j]) { if (n == nsn_id_pool[j]) {
@ -538,8 +553,9 @@ NsgIndex::InterInsert(float* data, unsigned n, std::vector<std::mutex>& mutex_ve
wait_for_link_pool.push_back(nsn); wait_for_link_pool.push_back(nsn);
} }
} }
if (duplicate) if (duplicate) {
continue; continue;
}
// original: (neighbor) <------- (current) // original: (neighbor) <------- (current)
// after: (neighbor) -------> (current) // after: (neighbor) -------> (current)
@ -570,8 +586,9 @@ NsgIndex::InterInsert(float* data, unsigned n, std::vector<std::mutex>& mutex_ve
if (nsn_dist_pool[j] == -1) { if (nsn_dist_pool[j] == -1) {
nsn_id_pool.push_back(current_as_neighbor.id); nsn_id_pool.push_back(current_as_neighbor.id);
nsn_dist_pool[j] = current_as_neighbor.distance; nsn_dist_pool[j] = current_as_neighbor.distance;
if (j + 1 < out_degree) if (j + 1 < out_degree) {
nsn_dist_pool[j + 1] = -1; nsn_dist_pool[j + 1] = -1;
}
break; break;
} }
} }
@ -594,16 +611,16 @@ NsgIndex::SelectEdge(float* data, unsigned& cursor, std::vector<Neighbor>& sort_
while (result.size() < out_degree && cursor < search_deepth && (++cursor) < pool.size()) { while (result.size() < out_degree && cursor < search_deepth && (++cursor) < pool.size()) {
auto& p = pool[cursor]; auto& p = pool[cursor];
bool should_link = true; bool should_link = true;
for (size_t t = 0; t < result.size(); ++t) { for (auto& t : result) {
float dist = distance_->Compare(data + dimension * result[t].id, data + dimension * p.id, dimension); float dist = distance_->Compare(data + dimension * t.id, data + dimension * p.id, dimension);
if (dist < p.distance) { if (dist < p.distance) {
should_link = false; should_link = false;
break; break;
} }
} }
if (should_link) if (should_link) {
result.push_back(p); result.push_back(p);
}
} }
} }
@ -636,16 +653,17 @@ NsgIndex::DFS(size_t root, boost::dynamic_bitset<>& has_linked, int64_t& linked_
while (!s.empty()) { while (!s.empty()) {
size_t next = ntotal + 1; size_t next = ntotal + 1;
for (unsigned i = 0; i < nsg[start].size(); i++) { for (auto i : nsg[start]) {
if (has_linked[nsg[start][i]] == false) { // if not link if (has_linked[i] == false) { // if not link
next = nsg[start][i]; next = i;
break; break;
} }
} }
if (next == (ntotal + 1)) { if (next == (ntotal + 1)) {
s.pop(); s.pop();
if (s.empty()) if (s.empty()) {
break; break;
}
start = s.top(); start = s.top();
continue; continue;
} }
@ -667,8 +685,9 @@ NsgIndex::FindUnconnectedNode(float* data, boost::dynamic_bitset<>& has_linked,
} }
} }
if (id == ntotal) if (id == ntotal) {
return; // No Unlinked Node return; // No Unlinked Node
}
// search unlinked-node's neighbor // search unlinked-node's neighbor
std::vector<Neighbor> tmp, pool; std::vector<Neighbor> tmp, pool;
@ -676,9 +695,9 @@ NsgIndex::FindUnconnectedNode(float* data, boost::dynamic_bitset<>& has_linked,
std::sort(pool.begin(), pool.end()); std::sort(pool.begin(), pool.end());
size_t found = 0; size_t found = 0;
for (size_t i = 0; i < pool.size(); i++) { // find nearest neighbor and add unlinked-node as its neighbor for (auto node : pool) { // find nearest neighbor and add unlinked-node as its neighbor
if (has_linked[pool[i].id]) { if (has_linked[node.id]) {
root = pool[i].id; root = node.id;
found = 1; found = 1;
break; break;
} }
@ -850,12 +869,13 @@ NsgIndex::Search(const float* query, float* data, const unsigned& nq, const unsi
bool is_ip = (metric_type == Metric_Type::Metric_Type_IP); bool is_ip = (metric_type == Metric_Type::Metric_Type_IP);
for (unsigned int i = 0; i < nq; ++i) { for (unsigned int i = 0; i < nq; ++i) {
unsigned int pos = 0; unsigned int pos = 0;
for (unsigned int j = 0; j < resset[i].size(); ++j) { for (auto node : resset[i]) {
if (pos >= k) if (pos >= k) {
break; // already top k break; // already top k
if (!bitset || !bitset->test((faiss::ConcurrentBitset::id_type_t)resset[i][j].id)) { }
ids[i * k + pos] = ids_[resset[i][j].id]; if (!bitset || !bitset->test(node.id)) {
dist[i * k + pos] = is_ip ? -resset[i][j].distance : resset[i][j].distance; ids[i * k + pos] = ids_[node.id];
dist[i * k + pos] = is_ip ? -node.distance : node.distance;
++pos; ++pos;
} }
} }
@ -880,11 +900,11 @@ NsgIndex::GetSize() {
ret += ntotal * dimension * sizeof(float); ret += ntotal * dimension * sizeof(float);
ret += ntotal * sizeof(int64_t); ret += ntotal * sizeof(int64_t);
ret += sizeof(*distance_); ret += sizeof(*distance_);
for (size_t i = 0; i < nsg.size(); ++i) { for (auto& v : nsg) {
ret += nsg[i].size() * sizeof(node_t); ret += v.size() * sizeof(node_t);
} }
for (size_t i = 0; i < knng.size(); ++i) { for (auto& v : knng) {
ret += knng[i].size() * sizeof(node_t); ret += v.size() * sizeof(node_t);
} }
return ret; return ret;
} }

View File

@ -44,7 +44,7 @@ using Graph = std::vector<std::vector<node_t>>;
class NsgIndex { class NsgIndex {
public: public:
enum Metric_Type { enum Metric_Type {
Metric_Type_L2, Metric_Type_L2 = 0,
Metric_Type_IP, Metric_Type_IP,
}; };

View File

@ -29,7 +29,7 @@ InsertIntoPool(Neighbor* addr, unsigned K, Neighbor nn) {
int left = 0, right = K - 1; int left = 0, right = K - 1;
if (addr[left].distance > nn.distance) { if (addr[left].distance > nn.distance) {
//>> Fix: memmove overflow, dump when vector<Neighbor> deconstruct //>> Fix: memmove overflow, dump when vector<Neighbor> deconstruct
memmove((char*)&addr[left + 1], &addr[left], (K - 1) * sizeof(Neighbor)); memmove(&addr[left + 1], &addr[left], (K - 1) * sizeof(Neighbor));
addr[left] = nn; addr[left] = nn;
return left; return left;
} }
@ -39,29 +39,33 @@ InsertIntoPool(Neighbor* addr, unsigned K, Neighbor nn) {
} }
while (left < right - 1) { while (left < right - 1) {
int mid = (left + right) / 2; int mid = (left + right) / 2;
if (addr[mid].distance > nn.distance) if (addr[mid].distance > nn.distance) {
right = mid; right = mid;
else } else {
left = mid; left = mid;
}
} }
// check equal ID // check equal ID
while (left > 0) { while (left > 0) {
if (addr[left].distance < nn.distance) // pos is right if (addr[left].distance < nn.distance) { // pos is right
break; break;
if (addr[left].id == nn.id) }
if (addr[left].id == nn.id) {
return K + 1; return K + 1;
}
left--; left--;
} }
if (addr[left].id == nn.id || addr[right].id == nn.id) if (addr[left].id == nn.id || addr[right].id == nn.id) {
return K + 1; return K + 1;
}
//>> Fix: memmove overflow, dump when vector<Neighbor> deconstruct //>> Fix: memmove overflow, dump when vector<Neighbor> deconstruct
memmove((char*)&addr[right + 1], &addr[right], (K - 1 - right) * sizeof(Neighbor)); memmove(&addr[right + 1], &addr[right], (K - 1 - right) * sizeof(Neighbor));
addr[right] = nn; addr[right] = nn;
return right; return right;
} }
}; // namespace impl } // namespace impl
} // namespace knowhere } // namespace knowhere
} // namespace milvus } // namespace milvus

View File

@ -27,7 +27,7 @@ write_index(NsgIndex* index, MemoryIOWriter& writer) {
writer(index->ids_, sizeof(int64_t) * index->ntotal, 1); writer(index->ids_, sizeof(int64_t) * index->ntotal, 1);
for (unsigned i = 0; i < index->ntotal; ++i) { for (unsigned i = 0; i < index->ntotal; ++i) {
auto neighbor_num = (node_t)index->nsg[i].size(); auto neighbor_num = static_cast<node_t>(index->nsg[i].size());
writer(&neighbor_num, sizeof(node_t), 1); writer(&neighbor_num, sizeof(node_t), 1);
writer(index->nsg[i].data(), neighbor_num * sizeof(node_t), 1); writer(index->nsg[i].data(), neighbor_num * sizeof(node_t), 1);
} }
@ -41,7 +41,7 @@ read_index(MemoryIOReader& reader) {
reader(&metric, sizeof(int32_t), 1); reader(&metric, sizeof(int32_t), 1);
reader(&ntotal, sizeof(size_t), 1); reader(&ntotal, sizeof(size_t), 1);
reader(&dimension, sizeof(size_t), 1); reader(&dimension, sizeof(size_t), 1);
auto index = new NsgIndex(dimension, ntotal, (impl::NsgIndex::Metric_Type)metric); auto index = new NsgIndex(dimension, ntotal, static_cast<NsgIndex::Metric_Type>(metric));
reader(&index->navigation_point, sizeof(index->navigation_point), 1); reader(&index->navigation_point, sizeof(index->navigation_point), 1);
// index->ori_data_ = new float[index->ntotal * index->dimension]; // index->ori_data_ = new float[index->ntotal * index->dimension];

View File

@ -158,7 +158,7 @@ install(TARGETS test_binaryivf DESTINATION unittest)
################################################################################ ################################################################################
#<NSG-TEST> #<NSG-TEST>
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) add_definitions(-std=c++11 -O3 -march=native -Werror -DINFO)
find_package(OpenMP REQUIRED) find_package(OpenMP REQUIRED)
if (OpenMP_FOUND) if (OpenMP_FOUND)

View File

@ -149,7 +149,7 @@ Segment::DeleteEntity(std::vector<offset_t>& offsets) {
return Status::OK(); return Status::OK();
} }
// sort offset in descendant // sort offset in descendant
std::sort(offsets.begin(), offsets.end(), std::greater<offset_t>()); std::sort(offsets.begin(), offsets.end(), std::greater<>());
// delete entity data from max offset to min offset // delete entity data from max offset to min offset
for (auto& pair : fixed_fields_) { for (auto& pair : fixed_fields_) {

View File

@ -26,7 +26,7 @@ class IOWriter {
InOpen(const std::string& name) = 0; InOpen(const std::string& name) = 0;
virtual void virtual void
Write(void* ptr, int64_t size) = 0; Write(const void* ptr, int64_t size) = 0;
virtual void virtual void
Seekp(int64_t pos) = 0; Seekp(int64_t pos) = 0;

View File

@ -30,8 +30,8 @@ DiskIOWriter::InOpen(const std::string& name) {
} }
void void
DiskIOWriter::Write(void* ptr, int64_t size) { DiskIOWriter::Write(const void* ptr, int64_t size) {
fs_.write(reinterpret_cast<char*>(ptr), size); fs_.write(reinterpret_cast<const char*>(ptr), size);
len_ += size; len_ += size;
} }

View File

@ -40,7 +40,7 @@ class DiskIOWriter : public IOWriter {
InOpen(const std::string& name) override; InOpen(const std::string& name) override;
void void
Write(void* ptr, int64_t size) override; Write(const void* ptr, int64_t size) override;
void void
Seekp(int64_t pos) override; Seekp(int64_t pos) override;

View File

@ -24,8 +24,8 @@ S3IOWriter::open(const std::string& name) {
} }
void void
S3IOWriter::write(void* ptr, int64_t size) { S3IOWriter::write(const void* ptr, int64_t size) {
buffer_ += std::string(reinterpret_cast<char*>(ptr), size); buffer_ += std::string(reinterpret_cast<const char*>(ptr), size);
len_ += size; len_ += size;
} }

View File

@ -36,7 +36,7 @@ class S3IOWriter : public IOWriter {
Open(const std::string& name) override; Open(const std::string& name) override;
void void
Write(void* ptr, int64_t size) override; Write(const void* ptr, int64_t size) override;
int64_t int64_t
Length() override; Length() override;

View File

@ -182,9 +182,9 @@ DBTest::SetUp() {
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0));
auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto default_conn = milvus::scheduler::Connection("IO", 500.0);
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("disk", "cpu", default_conn);
#ifdef MILVUS_GPU_VERSION #ifdef MILVUS_GPU_VERSION
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0));
res_mgr->Connect("cpu", "0", PCIE); res_mgr->Connect("cpu", "0", PCIE);
#endif #endif
@ -264,9 +264,9 @@ SchedulerTest::SetUp() {
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0));
auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto default_conn = milvus::scheduler::Connection("IO", 500.0);
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("disk", "cpu", default_conn);
#ifdef MILVUS_GPU_VERSION #ifdef MILVUS_GPU_VERSION
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0));
res_mgr->Connect("cpu", "0", PCIE); res_mgr->Connect("cpu", "0", PCIE);
#endif #endif

View File

@ -32,7 +32,7 @@ TEST_F(StorageTest, S3_CLIENT_TEST) {
const std::string filename_dummy = "/tmp/test_file_dummy"; const std::string filename_dummy = "/tmp/test_file_dummy";
const std::string filename_out = "/tmp/test_file_out"; const std::string filename_out = "/tmp/test_file_out";
const std::string objname = "/tmp/test_obj"; const std::string objname = "/tmp/test_obj";
const std::string objname_dummy = "/tmp/test_obj_dummy"; // const std::string objname_dummy = "/tmp/test_obj_dummy";
const std::string content = "abcdefghijklmnopqrstuvwxyz"; const std::string content = "abcdefghijklmnopqrstuvwxyz";
auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();