milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

modify bitset ref to value (#4697) 

Signed-off-by: shengjun.li <shengjun.li@zilliz.com>
pull/4699/head
shengjun.li 2021-02-05 21:22:18 +08:00 committed by GitHub
parent afe19a1a66
commit a3d4158fc6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
117 changed files with 277 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp
View File

@ -109,7 +109,7 @@ IndexAnnoy::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IndexAnnoy::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IndexAnnoy::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }

2
core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h
View File

@ -48,7 +48,7 @@ class IndexAnnoy : public VecIndex {
} }
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

6
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp
View File

@ -42,7 +42,7 @@ BinaryIDMAP::Load(const BinarySet& index_binary) {
} }
DatasetPtr DatasetPtr
BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize"); KNOWHERE_THROW_MSG("index not initialize");
} }
@ -67,7 +67,7 @@ BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const fa
DynamicResultSegment DynamicResultSegment
BinaryIDMAP::QueryByDistance(const milvus::knowhere::DatasetPtr& dataset, const milvus::knowhere::Config& config, BinaryIDMAP::QueryByDistance(const milvus::knowhere::DatasetPtr& dataset, const milvus::knowhere::Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize"); KNOWHERE_THROW_MSG("index not initialize");
} }
@ -151,7 +151,7 @@ BinaryIDMAP::GetRawVectors() {
void void
BinaryIDMAP::QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, BinaryIDMAP::QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels,
const Config& config, const faiss::BitsetView& bitset) { const Config& config, const faiss::BitsetView bitset) {
// assign the metric type // assign the metric type
index_->metric_type = GetMetricType(config[Metric::TYPE].get<std::string>()); index_->metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());

6
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h
View File

@ -45,10 +45,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr&, const Config&, const faiss::BitsetView& bitset) override; Query(const DatasetPtr&, const Config&, const faiss::BitsetView bitset) override;
DynamicResultSegment DynamicResultSegment
QueryByDistance(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView& bitset); QueryByDistance(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView bitset);
int64_t int64_t
Count() override; Count() override;
@ -67,7 +67,7 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex {
protected: protected:
virtual void virtual void
QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config, QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset); const faiss::BitsetView bitset);
}; };
using BinaryIDMAPPtr = std::shared_ptr<BinaryIDMAP>; using BinaryIDMAPPtr = std::shared_ptr<BinaryIDMAP>;

4
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp
View File

@ -51,7 +51,7 @@ BinaryIVF::Load(const BinarySet& index_binary) {
} }
DatasetPtr DatasetPtr
BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_ || !index_->is_trained) { if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }
@ -171,7 +171,7 @@ BinaryIVF::GenParams(const Config& config) {
void void
BinaryIVF::QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config, BinaryIVF::QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto params = GenParams(config); auto params = GenParams(config);
auto ivf_index = dynamic_cast<faiss::IndexBinaryIVF*>(index_.get()); auto ivf_index = dynamic_cast<faiss::IndexBinaryIVF*>(index_.get());
ivf_index->nprobe = params->nprobe; ivf_index->nprobe = params->nprobe;

4
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h
View File

@ -49,7 +49,7 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;
@ -72,7 +72,7 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex {
virtual void virtual void
QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config, QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset); const faiss::BitsetView bitset);
}; };
using BinaryIVFIndexPtr = std::shared_ptr<BinaryIVF>; using BinaryIVFIndexPtr = std::shared_ptr<BinaryIVF>;

2
core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp
View File

@ -138,7 +138,7 @@ IndexHNSW::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IndexHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IndexHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }

2
core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.h
View File

@ -41,7 +41,7 @@ class IndexHNSW : public VecIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

6
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
View File

@ -76,7 +76,7 @@ IDMAP::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize"); KNOWHERE_THROW_MSG("index not initialize");
} }
@ -100,7 +100,7 @@ IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::B
DynamicResultSegment DynamicResultSegment
IDMAP::QueryByDistance(const milvus::knowhere::DatasetPtr& dataset, const milvus::knowhere::Config& config, IDMAP::QueryByDistance(const milvus::knowhere::DatasetPtr& dataset, const milvus::knowhere::Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize"); KNOWHERE_THROW_MSG("index not initialize");
} }
@ -179,7 +179,7 @@ IDMAP::GetRawVectors() {
void void
IDMAP::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, IDMAP::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
// assign the metric type // assign the metric type
index_->metric_type = GetMetricType(config[Metric::TYPE].get<std::string>()); index_->metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
index_->search(n, data, k, distances, labels, bitset); index_->search(n, data, k, distances, labels, bitset);

6
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h
View File

@ -44,10 +44,10 @@ class IDMAP : public VecIndex, public FaissBaseIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr&, const Config&, const faiss::BitsetView&) override; Query(const DatasetPtr&, const Config&, const faiss::BitsetView) override;
DynamicResultSegment DynamicResultSegment
QueryByDistance(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView& bitset); QueryByDistance(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView bitset);
int64_t int64_t
Count() override; Count() override;
@ -68,7 +68,7 @@ class IDMAP : public VecIndex, public FaissBaseIndex {
protected: protected:
virtual void virtual void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView&); QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView);
}; };
using IDMAPPtr = std::shared_ptr<IDMAP>; using IDMAPPtr = std::shared_ptr<IDMAP>;

4
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
View File

@ -94,7 +94,7 @@ IVF::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IVF::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IVF::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_ || !index_->is_trained) { if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }
@ -304,7 +304,7 @@ IVF::GenParams(const Config& config) {
void void
IVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, IVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto params = GenParams(config); auto params = GenParams(config);
auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get()); auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get());
ivf_index->nprobe = std::min(params->nprobe, ivf_index->invlists->nlist); ivf_index->nprobe = std::min(params->nprobe, ivf_index->invlists->nlist);

4
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h
View File

@ -49,7 +49,7 @@ class IVF : public VecIndex, public FaissBaseIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr&, const Config&, const faiss::BitsetView&) override; Query(const DatasetPtr&, const Config&, const faiss::BitsetView) override;
#if 0 #if 0
DatasetPtr DatasetPtr
@ -90,7 +90,7 @@ class IVF : public VecIndex, public FaissBaseIndex {
GenParams(const Config&); GenParams(const Config&);
virtual void virtual void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView&); QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView);
void void
SealImpl() override; SealImpl() override;

2
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFHNSW.cpp
View File

@ -115,7 +115,7 @@ IVFHNSW::UpdateIndexSize() {
void void
IVFHNSW::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, IVFHNSW::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto params = GenParams(config); auto params = GenParams(config);
auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get()); auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get());
ivf_index->nprobe = std::min(params->nprobe, ivf_index->invlists->nlist); ivf_index->nprobe = std::min(params->nprobe, ivf_index->invlists->nlist);

2
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFHNSW.h
View File

@ -47,7 +47,7 @@ class IVFHNSW : public IVF {
protected: protected:
void void
QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) override; const faiss::BitsetView bitset) override;
}; };
using IVFHNSWPtr = std::shared_ptr<IVFHNSW>; using IVFHNSWPtr = std::shared_ptr<IVFHNSW>;

2
core/src/index/knowhere/knowhere/index/vector_index/IndexNGT.cpp
View File

@ -122,7 +122,7 @@ IndexNGT::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
#endif #endif
DatasetPtr DatasetPtr
IndexNGT::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IndexNGT::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize"); KNOWHERE_THROW_MSG("index not initialize");
} }

2
core/src/index/knowhere/knowhere/index/vector_index/IndexNGT.h
View File

@ -49,7 +49,7 @@ class IndexNGT : public VecIndex {
} }
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

2
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
View File

@ -99,7 +99,7 @@ IndexRHNSW::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IndexRHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IndexRHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_) { if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }

2
core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.h
View File

@ -51,7 +51,7 @@ class IndexRHNSW : public VecIndex, public FaissBaseIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

2
core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp
View File

@ -180,7 +180,7 @@ CPUSPTAGRNG::SetParameters(const Config& config) {
} }
DatasetPtr DatasetPtr
CPUSPTAGRNG::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { CPUSPTAGRNG::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
SetParameters(config); SetParameters(config);
float* p_data = (float*)dataset_ptr->Get<const void*>(meta::TENSOR); float* p_data = (float*)dataset_ptr->Get<const void*>(meta::TENSOR);

2
core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h
View File

@ -47,7 +47,7 @@ class CPUSPTAGRNG : public VecIndex {
} }
DatasetPtr DatasetPtr
Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) override; Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

2
core/src/index/knowhere/knowhere/index/vector_index/Statistics.h
View File

@ -147,7 +147,7 @@ class Statistics {
} }
void void
update_filter_percentage(const faiss::BitsetView& bitset) { update_filter_percentage(const faiss::BitsetView bitset) {
double fps = !bitset.empty() ? static_cast<double>(bitset.count_1()) / bitset.size() : 0.0; double fps = !bitset.empty() ? static_cast<double>(bitset.count_1()) / bitset.size() : 0.0;
filter_stat[static_cast<int>(fps * 100) / 5] += 1; filter_stat[static_cast<int>(fps * 100) / 5] += 1;
} }

2
core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h
View File

@ -47,7 +47,7 @@ class VecIndex : public Index {
AddWithoutIds(const DatasetPtr& dataset, const Config& config) = 0; AddWithoutIds(const DatasetPtr& dataset, const Config& config) = 0;
virtual DatasetPtr virtual DatasetPtr
Query(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView& bitset) = 0; Query(const DatasetPtr& dataset, const Config& config, const faiss::BitsetView bitset) = 0;
virtual int64_t virtual int64_t
Dim() = 0; Dim() = 0;

2
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
View File

@ -98,7 +98,7 @@ GPUIDMAP::GetRawVectors() {
void void
GPUIDMAP::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, GPUIDMAP::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
ResScope rs(res_, gpu_id_); ResScope rs(res_, gpu_id_);
// assign the metric type // assign the metric type

3
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.h
View File

@ -52,8 +52,7 @@ class GPUIDMAP : public IDMAP, public GPUIndex {
LoadImpl(const BinarySet&, const IndexType&) override; LoadImpl(const BinarySet&, const IndexType&) override;
void void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView) override;
const faiss::BitsetView& bitset) override;
}; };
using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>; using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>;

2
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp
View File

@ -135,7 +135,7 @@ GPUIVF::LoadImpl(const BinarySet& binary_set, const IndexType& type) {
void void
GPUIVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, GPUIVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_); auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_);
fiu_do_on("GPUIVF.search_impl.invald_index", device_index = nullptr); fiu_do_on("GPUIVF.search_impl.invald_index", device_index = nullptr);
if (device_index) { if (device_index) {

3
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.h
View File

@ -51,8 +51,7 @@ class GPUIVF : public IVF, public GPUIndex {
LoadImpl(const BinarySet&, const IndexType&) override; LoadImpl(const BinarySet&, const IndexType&) override;
void void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView) override;
const faiss::BitsetView& bitset) override;
}; };
using GPUIVFPtr = std::shared_ptr<GPUIVF>; using GPUIVFPtr = std::shared_ptr<GPUIVF>;

2
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
View File

@ -241,7 +241,7 @@ IVFSQHybrid::LoadImpl(const BinarySet& binary_set, const IndexType& type) {
void void
IVFSQHybrid::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, IVFSQHybrid::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
if (gpu_mode_ == 2) { if (gpu_mode_ == 2) {
GPUIVF::QueryImpl(n, data, k, distances, labels, config, bitset); GPUIVF::QueryImpl(n, data, k, distances, labels, config, bitset);
// index_->search(n, (float*)data, k, distances, labels); // index_->search(n, (float*)data, k, distances, labels);

3
core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.h
View File

@ -88,8 +88,7 @@ class IVFSQHybrid : public GPUIVFSQ {
LoadImpl(const BinarySet&, const IndexType&) override; LoadImpl(const BinarySet&, const IndexType&) override;
void void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView) override;
const faiss::BitsetView& bitset) override;
protected: protected:
int64_t gpu_mode_ = 0; // 0: CPU, 1: Hybrid, 2: GPU int64_t gpu_mode_ = 0; // 0: CPU, 1: Hybrid, 2: GPU

2
core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.cpp
View File

@ -858,7 +858,7 @@ NsgIndex::FindUnconnectedNode(float* data, boost::dynamic_bitset<>& has_linked,
void void
NsgIndex::Search(const float* query, float* data, const unsigned& nq, const unsigned& dim, const unsigned& k, NsgIndex::Search(const float* query, float* data, const unsigned& nq, const unsigned& dim, const unsigned& k,
float* dist, int64_t* ids, SearchParams& params, const faiss::BitsetView& bitset) { float* dist, int64_t* ids, SearchParams& params, const faiss::BitsetView bitset) {
std::vector<std::vector<Neighbor>> resset(nq); std::vector<std::vector<Neighbor>> resset(nq);
TimeRecorder rc("NsgIndex::search", 1); TimeRecorder rc("NsgIndex::search", 1);

2
core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.h
View File

@ -84,7 +84,7 @@ class NsgIndex {
void void
Search(const float* query, float* data, const unsigned& nq, const unsigned& dim, const unsigned& k, float* dist, Search(const float* query, float* data, const unsigned& nq, const unsigned& dim, const unsigned& k, float* dist,
int64_t* ids, SearchParams& params, const faiss::BitsetView& bitset = nullptr); int64_t* ids, SearchParams& params, const faiss::BitsetView bitset);
int64_t int64_t
GetSize(); GetSize();

4
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
View File

@ -137,7 +137,7 @@ IVF_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
} }
DatasetPtr DatasetPtr
IVF_NM::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { IVF_NM::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_ || !index_->is_trained) { if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }
@ -306,7 +306,7 @@ IVF_NM::GenParams(const Config& config) {
void void
IVF_NM::QueryImpl(int64_t n, const float* query, int64_t k, float* distances, int64_t* labels, const Config& config, IVF_NM::QueryImpl(int64_t n, const float* query, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto params = GenParams(config); auto params = GenParams(config);
auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get()); auto ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get());
ivf_index->nprobe = params->nprobe; ivf_index->nprobe = params->nprobe;

4
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h
View File

@ -50,7 +50,7 @@ class IVF_NM : public VecIndex, public OffsetBaseIndex {
AddWithoutIds(const DatasetPtr&, const Config&) override; AddWithoutIds(const DatasetPtr&, const Config&) override;
DatasetPtr DatasetPtr
Query(const DatasetPtr&, const Config&, const faiss::BitsetView& bitset) override; Query(const DatasetPtr&, const Config&, const faiss::BitsetView bitset) override;
#if 0 #if 0
DatasetPtr DatasetPtr
@ -91,7 +91,7 @@ class IVF_NM : public VecIndex, public OffsetBaseIndex {
GenParams(const Config&); GenParams(const Config&);
virtual void virtual void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView& bitset); QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView bitset);
void void
SealImpl() override; SealImpl() override;

2
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp
View File

@ -76,7 +76,7 @@ NSG_NM::Load(const BinarySet& index_binary) {
} }
DatasetPtr DatasetPtr
NSG_NM::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView& bitset) { NSG_NM::Query(const DatasetPtr& dataset_ptr, const Config& config, const faiss::BitsetView bitset) {
if (!index_ || !index_->is_trained) { if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained"); KNOWHERE_THROW_MSG("index not initialize or trained");
} }

2
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.h
View File

@ -54,7 +54,7 @@ class NSG_NM : public VecIndex {
} }
DatasetPtr DatasetPtr
Query(const DatasetPtr&, const Config&, const faiss::BitsetView& bitset) override; Query(const DatasetPtr&, const Config&, const faiss::BitsetView bitset) override;
int64_t int64_t
Count() override; Count() override;

2
core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp
View File

@ -117,7 +117,7 @@ GPUIVF_NM::SerializeImpl(const IndexType& type) {
void void
GPUIVF_NM::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config, GPUIVF_NM::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_); auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_);
fiu_do_on("GPUIVF_NM.search_impl.invald_index", device_index = nullptr); fiu_do_on("GPUIVF_NM.search_impl.invald_index", device_index = nullptr);
if (device_index) { if (device_index) {

3
core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.h
View File

@ -51,8 +51,7 @@ class GPUIVF_NM : public IVF, public GPUIndex {
SerializeImpl(const IndexType&) override; SerializeImpl(const IndexType&) override;
void void
QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, QueryImpl(int64_t, const float*, int64_t, float*, int64_t*, const Config&, const faiss::BitsetView) override;
const faiss::BitsetView& bitset) override;
protected: protected:
uint8_t* arranged_data; uint8_t* arranged_data;

2
core/src/index/thirdparty/NGT/lib/NGT/Graph.cpp vendored
View File

@ -599,7 +599,7 @@ NeighborhoodGraph::setupSeeds(NGT::SearchContainer &sc, ObjectDistances &seeds,
} }
// for milvus // for milvus
void NeighborhoodGraph::search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView& bitset) void NeighborhoodGraph::search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView bitset)
{ {
if (sc.explorationCoefficient == 0.0) if (sc.explorationCoefficient == 0.0)
{ {

2
core/src/index/thirdparty/NGT/lib/NGT/Graph.h vendored
View File

@ -698,7 +698,7 @@ namespace NGT {
void search(NGT::SearchContainer &sc, ObjectDistances &seeds); void search(NGT::SearchContainer &sc, ObjectDistances &seeds);
// for milvus // for milvus
void search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView&bitset); void search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView bitset);
#ifdef NGT_GRAPH_READ_ONLY_GRAPH #ifdef NGT_GRAPH_READ_ONLY_GRAPH
template <typename COMPARATOR, typename CHECK_LIST> void searchReadOnlyGraph(NGT::SearchContainer &sc, ObjectDistances &seeds); template <typename COMPARATOR, typename CHECK_LIST> void searchReadOnlyGraph(NGT::SearchContainer &sc, ObjectDistances &seeds);

10
core/src/index/thirdparty/NGT/lib/NGT/Index.h vendored
View File

@ -655,7 +655,7 @@ public:
virtual void linearSearch(NGT::SearchContainer & sc) { getIndex().linearSearch(sc); } virtual void linearSearch(NGT::SearchContainer & sc) { getIndex().linearSearch(sc); }
virtual void linearSearch(NGT::SearchQuery & sc) { getIndex().linearSearch(sc); } virtual void linearSearch(NGT::SearchQuery & sc) { getIndex().linearSearch(sc); }
// for milvus // for milvus
virtual void search(NGT::SearchContainer & sc, const faiss::BitsetView&bitset) { getIndex().search(sc, bitset); } virtual void search(NGT::SearchContainer & sc, const faiss::BitsetView bitset) { getIndex().search(sc, bitset); }
virtual void search(NGT::SearchContainer & sc) { getIndex().search(sc); } virtual void search(NGT::SearchContainer & sc) { getIndex().search(sc); }
virtual void search(NGT::SearchQuery & sc) { getIndex().search(sc); } virtual void search(NGT::SearchQuery & sc) { getIndex().search(sc); }
virtual void search(NGT::SearchContainer & sc, ObjectDistances & seeds) { getIndex().search(sc, seeds); } virtual void search(NGT::SearchContainer & sc, ObjectDistances & seeds) { getIndex().search(sc, seeds); }
@ -1058,7 +1058,7 @@ public:
} }
// for milvus // for milvus
virtual void search(NGT::SearchContainer & sc, const faiss::BitsetView&bitset) virtual void search(NGT::SearchContainer & sc, const faiss::BitsetView bitset)
{ {
sc.distanceComputationCount = 0; sc.distanceComputationCount = 0;
sc.visitCount = 0; sc.visitCount = 0;
@ -1586,7 +1586,7 @@ protected:
} }
// for milvus // for milvus
virtual void search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView&bitset) virtual void search(NGT::SearchContainer & sc, ObjectDistances & seeds, const faiss::BitsetView bitset)
{ {
if (sc.size == 0) if (sc.size == 0)
{ {
@ -2147,7 +2147,7 @@ public:
// for milvus // for milvus
void void
getSeedsFromTree(NGT::SearchContainer& sc, ObjectDistances& seeds, const faiss::BitsetView& bitset) { getSeedsFromTree(NGT::SearchContainer& sc, ObjectDistances& seeds, const faiss::BitsetView bitset) {
DVPTree::SearchContainer tso(sc.object); DVPTree::SearchContainer tso(sc.object);
tso.mode = DVPTree::SearchContainer::SearchLeaf; tso.mode = DVPTree::SearchContainer::SearchLeaf;
tso.radius = 0.0; tso.radius = 0.0;
@ -2204,7 +2204,7 @@ public:
} }
// for milvus // for milvus
void search(NGT::SearchContainer & sc, const faiss::BitsetView&bitset) void search(NGT::SearchContainer & sc, const faiss::BitsetView bitset)
{ {
sc.distanceComputationCount = 0; sc.distanceComputationCount = 0;
sc.visitCount = 0; sc.visitCount = 0;

2
core/src/index/thirdparty/NGT/lib/NGT/Tree.h vendored
View File

@ -263,7 +263,7 @@ namespace NGT {
// for milvus // for milvus
void void
getObjectIDsFromLeaf(Node::ID nid, ObjectDistances& rl, const faiss::BitsetView& bitset) { getObjectIDsFromLeaf(Node::ID nid, ObjectDistances& rl, const faiss::BitsetView bitset) {
LeafNode& ln = *(LeafNode*)getNode(nid); LeafNode& ln = *(LeafNode*)getNode(nid);
rl.clear(); rl.clear();
ObjectDistance r; ObjectDistance r;

10
core/src/index/thirdparty/annoy/src/annoylib.h vendored
View File

@ -839,9 +839,9 @@ class AnnoyIndexInterface {
virtual bool load_index(void* index_data, const int64_t& index_size, char** error = nullptr) = 0; virtual bool load_index(void* index_data, const int64_t& index_size, char** error = nullptr) = 0;
virtual T get_distance(S i, S j) const = 0; virtual T get_distance(S i, S j) const = 0;
virtual void get_nns_by_item(S item, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances, virtual void get_nns_by_item(S item, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances,
const faiss::BitsetView& bitset = nullptr) const = 0; const faiss::BitsetView bitset = nullptr) const = 0;
virtual void get_nns_by_vector(const T* w, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances, virtual void get_nns_by_vector(const T* w, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances,
const faiss::BitsetView& bitset = nullptr) const = 0; const faiss::BitsetView bitset = nullptr) const = 0;
virtual S get_n_items() const = 0; virtual S get_n_items() const = 0;
virtual S get_dim() const = 0; virtual S get_dim() const = 0;
virtual S get_n_trees() const = 0; virtual S get_n_trees() const = 0;
@ -1178,14 +1178,14 @@ public:
} }
void get_nns_by_item(S item, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances, void get_nns_by_item(S item, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances,
const faiss::BitsetView& bitset) const { const faiss::BitsetView bitset) const {
// TODO: handle OOB // TODO: handle OOB
const Node* m = _get(item); const Node* m = _get(item);
_get_all_nns(m->v, n, search_k, result, distances, bitset); _get_all_nns(m->v, n, search_k, result, distances, bitset);
} }
void get_nns_by_vector(const T* w, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances, void get_nns_by_vector(const T* w, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances,
const faiss::BitsetView& bitset) const { const faiss::BitsetView bitset) const {
_get_all_nns(w, n, search_k, result, distances, bitset); _get_all_nns(w, n, search_k, result, distances, bitset);
} }
@ -1335,7 +1335,7 @@ protected:
} }
void _get_all_nns(const T* v, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances, void _get_all_nns(const T* v, size_t n, int64_t search_k, vector<S>* result, vector<T>* distances,
const faiss::BitsetView& bitset) const { const faiss::BitsetView bitset) const {
Node* v_node = (Node *)alloca(_s); Node* v_node = (Node *)alloca(_s);
D::template zero_value<Node>(v_node); D::template zero_value<Node>(v_node);
memcpy(v_node->v, v, sizeof(T) * _f); memcpy(v_node->v, v, sizeof(T) * _f);

6
core/src/index/thirdparty/faiss/Index.cpp vendored
View File

@ -31,7 +31,7 @@ void Index::train(idx_t /*n*/, const float* /*x*/) {
void Index::range_search (idx_t , const float *, float, void Index::range_search (idx_t , const float *, float,
RangeSearchResult *, RangeSearchResult *,
const BitsetView&) const const BitsetView) const
{ {
FAISS_THROW_MSG ("range search not implemented"); FAISS_THROW_MSG ("range search not implemented");
} }
@ -62,12 +62,12 @@ void Index::add_with_ids_without_codes(idx_t n, const float* x, const idx_t* xid
} }
#if 0 #if 0
void Index::get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView& bitset) { void Index::get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView bitset) {
FAISS_THROW_MSG ("get_vector_by_id not implemented for this type of index"); FAISS_THROW_MSG ("get_vector_by_id not implemented for this type of index");
} }
void Index::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, void Index::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset) { const BitsetView bitset) {
FAISS_THROW_MSG ("search_by_id not implemented for this type of index"); FAISS_THROW_MSG ("search_by_id not implemented for this type of index");
} }
#endif #endif

8
core/src/index/thirdparty/faiss/Index.h vendored
View File

@ -129,7 +129,7 @@ struct Index {
*/ */
virtual void search (idx_t n, const float *x, idx_t k, virtual void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const = 0; const BitsetView bitset = nullptr) const = 0;
#if 0 #if 0
/** query n raw vectors from the index by ids. /** query n raw vectors from the index by ids.
@ -141,7 +141,7 @@ struct Index {
* @param x output raw vectors, size n * d * @param x output raw vectors, size n * d
* @param bitset flags to check the validity of vectors * @param bitset flags to check the validity of vectors
*/ */
virtual void get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView& bitset = nullptr); virtual void get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView bitset = nullptr);
/** query n vectors of dimension d to the index by ids. /** query n vectors of dimension d to the index by ids.
* *
@ -154,7 +154,7 @@ struct Index {
* @param bitset flags to check the validity of vectors * @param bitset flags to check the validity of vectors
*/ */
virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr); const BitsetView bitset = nullptr);
#endif #endif
/** query n vectors of dimension d to the index. /** query n vectors of dimension d to the index.
@ -169,7 +169,7 @@ struct Index {
*/ */
virtual void range_search (idx_t n, const float *x, float radius, virtual void range_search (idx_t n, const float *x, float radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
/** return the indexes of the k vectors closest to the query x. /** return the indexes of the k vectors closest to the query x.
* *

2
core/src/index/thirdparty/faiss/Index2Layer.cpp vendored
View File

@ -166,7 +166,7 @@ void Index2Layer::search(
idx_t /*k*/, idx_t /*k*/,
float* /*distances*/, float* /*distances*/,
idx_t* /*labels*/, idx_t* /*labels*/,
const BitsetView&) const { const BitsetView) const {
FAISS_THROW_MSG("not implemented"); FAISS_THROW_MSG("not implemented");
} }

2
core/src/index/thirdparty/faiss/Index2Layer.h vendored
View File

@ -61,7 +61,7 @@ struct Index2Layer: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override; void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;

6
core/src/index/thirdparty/faiss/IndexBinary.cpp vendored
View File

@ -22,7 +22,7 @@ void IndexBinary::train(idx_t, const uint8_t *) {
void IndexBinary::range_search(idx_t, const uint8_t *, int, void IndexBinary::range_search(idx_t, const uint8_t *, int,
RangeSearchResult *, RangeSearchResult *,
const BitsetView&) const { const BitsetView) const {
FAISS_THROW_MSG("range search not implemented"); FAISS_THROW_MSG("range search not implemented");
} }
@ -37,12 +37,12 @@ void IndexBinary::add_with_ids(idx_t, const uint8_t *, const idx_t *) {
} }
#if 0 #if 0
void IndexBinary::get_vector_by_id (idx_t n, const idx_t *xid, uint8_t *x, const BitsetView& bitset) { void IndexBinary::get_vector_by_id (idx_t n, const idx_t *xid, uint8_t *x, const BitsetView bitset) {
FAISS_THROW_MSG("get_vector_by_id not implemented for this type of index"); FAISS_THROW_MSG("get_vector_by_id not implemented for this type of index");
} }
void IndexBinary::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, void IndexBinary::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels,
const BitsetView& bitset) { const BitsetView bitset) {
FAISS_THROW_MSG("search_by_id not implemented for this type of index"); FAISS_THROW_MSG("search_by_id not implemented for this type of index");
} }
#endif #endif

8
core/src/index/thirdparty/faiss/IndexBinary.h vendored
View File

@ -97,7 +97,7 @@ struct IndexBinary {
*/ */
virtual void search(idx_t n, const uint8_t *x, idx_t k, virtual void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const = 0; const BitsetView bitset = nullptr) const = 0;
#if 0 #if 0
/** Query n raw vectors from the index by ids. /** Query n raw vectors from the index by ids.
@ -109,7 +109,7 @@ struct IndexBinary {
* @param x output raw vectors, size n * d * @param x output raw vectors, size n * d
* @param bitset flags to check the validity of vectors * @param bitset flags to check the validity of vectors
*/ */
virtual void get_vector_by_id (idx_t n, const idx_t *xid, uint8_t *x, const BitsetView& bitset = nullptr); virtual void get_vector_by_id (idx_t n, const idx_t *xid, uint8_t *x, const BitsetView bitset = nullptr);
/** query n vectors of dimension d to the index by ids. /** query n vectors of dimension d to the index by ids.
* *
@ -122,7 +122,7 @@ struct IndexBinary {
* @param bitset flags to check the validity of vectors * @param bitset flags to check the validity of vectors
*/ */
virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr); const BitsetView bitset = nullptr);
#endif #endif
/** Query n vectors of dimension d to the index. /** Query n vectors of dimension d to the index.
@ -141,7 +141,7 @@ struct IndexBinary {
*/ */
virtual void range_search(idx_t n, const uint8_t *x, int radius, virtual void range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
/** Return the indexes of the k vectors closest to the query x. /** Return the indexes of the k vectors closest to the query x.
* *

6
core/src/index/thirdparty/faiss/IndexBinaryFlat.cpp vendored
View File

@ -40,7 +40,7 @@ void IndexBinaryFlat::reset() {
void IndexBinaryFlat::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryFlat::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) { if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) {
float *D = reinterpret_cast<float*>(distances); float *D = reinterpret_cast<float*>(distances);
@ -98,7 +98,7 @@ void IndexBinaryFlat::reconstruct(idx_t key, uint8_t *recons) const {
void IndexBinaryFlat::range_search(idx_t n, const uint8_t *x, int radius, void IndexBinaryFlat::range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_MSG("This interface is abandoned yet."); FAISS_THROW_MSG("This interface is abandoned yet.");
} }
@ -108,7 +108,7 @@ void IndexBinaryFlat::range_search(faiss::IndexBinary::idx_t n,
float radius, float radius,
std::vector<faiss::RangeSearchPartialResult*>& result, std::vector<faiss::RangeSearchPartialResult*>& result,
size_t buffer_size, size_t buffer_size,
const faiss::BitsetView& bitset) const faiss::BitsetView bitset)
{ {
switch (metric_type) { switch (metric_type) {
case METRIC_Jaccard: { case METRIC_Jaccard: {

6
core/src/index/thirdparty/faiss/IndexBinaryFlat.h vendored
View File

@ -40,16 +40,16 @@ struct IndexBinaryFlat : IndexBinary {
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void range_search(idx_t n, const uint8_t *x, int radius, void range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void range_search(idx_t n, const uint8_t *x, float radius, void range_search(idx_t n, const uint8_t *x, float radius,
std::vector<RangeSearchPartialResult*> &result, std::vector<RangeSearchPartialResult*> &result,
size_t buffer_size, size_t buffer_size,
const BitsetView& bitset = nullptr); // const override const BitsetView bitset = nullptr); // const override
void reconstruct(idx_t key, uint8_t *recons) const override; void reconstruct(idx_t key, uint8_t *recons) const override;

2
core/src/index/thirdparty/faiss/IndexBinaryFromFloat.cpp vendored
View File

@ -51,7 +51,7 @@ void IndexBinaryFromFloat::reset() {
void IndexBinaryFromFloat::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryFromFloat::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
constexpr idx_t bs = 32768; constexpr idx_t bs = 32768;
std::unique_ptr<float[]> xf(new float[bs * d]); std::unique_ptr<float[]> xf(new float[bs * d]);
std::unique_ptr<float[]> df(new float[bs * k]); std::unique_ptr<float[]> df(new float[bs * k]);

2
core/src/index/thirdparty/faiss/IndexBinaryFromFloat.h vendored
View File

@ -42,7 +42,7 @@ struct IndexBinaryFromFloat : IndexBinary {
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void train(idx_t n, const uint8_t *x) override; void train(idx_t n, const uint8_t *x) override;
}; };

2
core/src/index/thirdparty/faiss/IndexBinaryHNSW.cpp vendored
View File

@ -197,7 +197,7 @@ void IndexBinaryHNSW::train(idx_t n, const uint8_t *x)
void IndexBinaryHNSW::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryHNSW::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
#pragma omp parallel #pragma omp parallel
{ {

2
core/src/index/thirdparty/faiss/IndexBinaryHNSW.h vendored
View File

@ -46,7 +46,7 @@ struct IndexBinaryHNSW : IndexBinary {
/// entry point for search /// entry point for search
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct(idx_t key, uint8_t* recons) const override; void reconstruct(idx_t key, uint8_t* recons) const override;

8
core/src/index/thirdparty/faiss/IndexBinaryHash.cpp vendored
View File

@ -209,7 +209,7 @@ search_single_query(const IndexBinaryHash & index, const uint8_t *q,
void IndexBinaryHash::range_search(idx_t n, const uint8_t *x, int radius, void IndexBinaryHash::range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
size_t nlist = 0, ndis = 0, n0 = 0; size_t nlist = 0, ndis = 0, n0 = 0;
@ -237,7 +237,7 @@ void IndexBinaryHash::range_search(idx_t n, const uint8_t *x, int radius,
void IndexBinaryHash::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryHash::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
using HeapForL2 = CMax<int32_t, idx_t>; using HeapForL2 = CMax<int32_t, idx_t>;
@ -421,7 +421,7 @@ search_1_query_multihash(const IndexBinaryMultiHash & index, const uint8_t *xi,
void IndexBinaryMultiHash::range_search(idx_t n, const uint8_t *x, int radius, void IndexBinaryMultiHash::range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
size_t nlist = 0, ndis = 0, n0 = 0; size_t nlist = 0, ndis = 0, n0 = 0;
@ -449,7 +449,7 @@ void IndexBinaryMultiHash::range_search(idx_t n, const uint8_t *x, int radius,
void IndexBinaryMultiHash::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryMultiHash::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
using HeapForL2 = CMax<int32_t, idx_t>; using HeapForL2 = CMax<int32_t, idx_t>;

8
core/src/index/thirdparty/faiss/IndexBinaryHash.h vendored
View File

@ -52,11 +52,11 @@ struct IndexBinaryHash : IndexBinary {
void range_search(idx_t n, const uint8_t *x, int radius, void range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void display() const; void display() const;
size_t hashtable_size() const; size_t hashtable_size() const;
@ -105,11 +105,11 @@ struct IndexBinaryMultiHash: IndexBinary {
void range_search(idx_t n, const uint8_t *x, int radius, void range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
size_t hashtable_size() const; size_t hashtable_size() const;

22
core/src/index/thirdparty/faiss/IndexBinaryIVF.cpp vendored
View File

@ -148,7 +148,7 @@ void IndexBinaryIVF::set_direct_map_type (DirectMap::Type type)
void IndexBinaryIVF::search(idx_t n, const uint8_t *x, idx_t k, void IndexBinaryIVF::search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]); std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]);
std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n * nprobe]); std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n * nprobe]);
@ -173,7 +173,7 @@ void IndexBinaryIVF::search(idx_t n, const uint8_t *x, idx_t k,
} }
#if 0 #if 0
void IndexBinaryIVF::get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, const BitsetView& bitset) { void IndexBinaryIVF::get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, const BitsetView bitset) {
make_direct_map(true); make_direct_map(true);
/* only get vector by 1 id */ /* only get vector by 1 id */
@ -186,7 +186,7 @@ void IndexBinaryIVF::get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, con
} }
void IndexBinaryIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, void IndexBinaryIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels,
const BitsetView& bitset) { const BitsetView bitset) {
make_direct_map(true); make_direct_map(true);
auto x = new uint8_t[n * d]; auto x = new uint8_t[n * d];
@ -385,7 +385,7 @@ struct IVFBinaryScannerL2: BinaryInvertedListScanner {
const idx_t *ids, const idx_t *ids,
int32_t *simi, idx_t *idxi, int32_t *simi, idx_t *idxi,
size_t k, size_t k,
const BitsetView& bitset) const override const BitsetView bitset) const override
{ {
using C = CMax<int32_t, idx_t>; using C = CMax<int32_t, idx_t>;
@ -448,7 +448,7 @@ struct IVFBinaryScannerJaccard: BinaryInvertedListScanner {
const idx_t *ids, const idx_t *ids,
int32_t *simi, idx_t *idxi, int32_t *simi, idx_t *idxi,
size_t k, size_t k,
const BitsetView& bitset = nullptr) const override const BitsetView bitset = nullptr) const override
{ {
using C = CMax<float, idx_t>; using C = CMax<float, idx_t>;
float* psimi = (float*)simi; float* psimi = (float*)simi;
@ -521,7 +521,7 @@ void search_knn_hamming_heap(const IndexBinaryIVF& ivf,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
IndexIVFStats &index_ivf_stats, IndexIVFStats &index_ivf_stats,
const BitsetView& bitset = nullptr) const BitsetView bitset = nullptr)
{ {
long nprobe = params ? params->nprobe : ivf.nprobe; long nprobe = params ? params->nprobe : ivf.nprobe;
long max_codes = params ? params->max_codes : ivf.max_codes; long max_codes = params ? params->max_codes : ivf.max_codes;
@ -618,7 +618,7 @@ void search_knn_binary_dis_heap(const IndexBinaryIVF& ivf,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
IndexIVFStats &index_ivf_stats, IndexIVFStats &index_ivf_stats,
const BitsetView& bitset = nullptr) const BitsetView bitset = nullptr)
{ {
long nprobe = params ? params->nprobe : ivf.nprobe; long nprobe = params ? params->nprobe : ivf.nprobe;
long max_codes = params ? params->max_codes : ivf.max_codes; long max_codes = params ? params->max_codes : ivf.max_codes;
@ -705,7 +705,7 @@ void search_knn_hamming_count(const IndexBinaryIVF& ivf,
idx_t *labels, idx_t *labels,
const IVFSearchParameters *params, const IVFSearchParameters *params,
IndexIVFStats &index_ivf_stats, IndexIVFStats &index_ivf_stats,
const BitsetView& bitset = nullptr) { const BitsetView bitset = nullptr) {
const int nBuckets = ivf.d + 1; const int nBuckets = ivf.d + 1;
std::vector<int> all_counters(nx * nBuckets, 0); std::vector<int> all_counters(nx * nBuckets, 0);
std::unique_ptr<idx_t[]> all_ids_per_dis(new idx_t[nx * nBuckets * k]); std::unique_ptr<idx_t[]> all_ids_per_dis(new idx_t[nx * nBuckets * k]);
@ -803,7 +803,7 @@ void search_knn_hamming_count_1 (
idx_t *labels, idx_t *labels,
const IVFSearchParameters *params, const IVFSearchParameters *params,
IndexIVFStats &index_ivf_stats, IndexIVFStats &index_ivf_stats,
const BitsetView& bitset = nullptr) { const BitsetView bitset = nullptr) {
switch (ivf.code_size) { switch (ivf.code_size) {
#define HANDLE_CS(cs) \ #define HANDLE_CS(cs) \
case cs: \ case cs: \
@ -856,7 +856,7 @@ void IndexBinaryIVF::search_preassigned(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
const BitsetView& bitset const BitsetView bitset
) const { ) const {
if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) { if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) {
if (use_heap) { if (use_heap) {
@ -899,7 +899,7 @@ void IndexBinaryIVF::search_preassigned(idx_t n, const uint8_t *x, idx_t k,
void IndexBinaryIVF::range_search( void IndexBinaryIVF::range_search(
idx_t n, const uint8_t *x, int radius, idx_t n, const uint8_t *x, int radius,
RangeSearchResult *res, RangeSearchResult *res,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]); std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]);
std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n * nprobe]); std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n * nprobe]);

12
core/src/index/thirdparty/faiss/IndexBinaryIVF.h vendored
View File

@ -108,7 +108,7 @@ struct IndexBinaryIVF : IndexBinary {
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params=nullptr, const IVFSearchParameters *params=nullptr,
const BitsetView& bitset = nullptr const BitsetView bitset = nullptr
) const; ) const;
virtual BinaryInvertedListScanner *get_InvertedListScanner ( virtual BinaryInvertedListScanner *get_InvertedListScanner (
@ -116,20 +116,20 @@ struct IndexBinaryIVF : IndexBinary {
/** assign the vectors, then call search_preassign */ /** assign the vectors, then call search_preassign */
void search(idx_t n, const uint8_t *x, idx_t k, void search(idx_t n, const uint8_t *x, idx_t k,
int32_t *distances, idx_t *labels, const BitsetView& bitset = nullptr) const override; int32_t *distances, idx_t *labels, const BitsetView bitset = nullptr) const override;
#if 0 #if 0
/** get raw vectors by ids */ /** get raw vectors by ids */
void get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, const BitsetView& bitset = nullptr) override; void get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, const BitsetView bitset = nullptr) override;
void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) override; const BitsetView bitset = nullptr) override;
#endif #endif
void range_search(idx_t n, const uint8_t *x, int radius, void range_search(idx_t n, const uint8_t *x, int radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct(idx_t key, uint8_t *recons) const override; void reconstruct(idx_t key, uint8_t *recons) const override;
@ -230,7 +230,7 @@ struct BinaryInvertedListScanner {
const idx_t *ids, const idx_t *ids,
int32_t *distances, idx_t *labels, int32_t *distances, idx_t *labels,
size_t k, size_t k,
const BitsetView& bitset = nullptr) const = 0; const BitsetView bitset = nullptr) const = 0;
virtual void scan_codes_range (size_t n, virtual void scan_codes_range (size_t n,
const uint8_t *codes, const uint8_t *codes,

12
core/src/index/thirdparty/faiss/IndexFlat.cpp vendored
View File

@ -41,7 +41,7 @@ void IndexFlat::reset() {
void IndexFlat::search (idx_t n, const float *x, idx_t k, void IndexFlat::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
// we see the distances and labels as heaps // we see the distances and labels as heaps
@ -91,7 +91,7 @@ void IndexFlat::assign(idx_t n, const float * x, idx_t * labels, float* distance
void IndexFlat::range_search (idx_t n, const float *x, float radius, void IndexFlat::range_search (idx_t n, const float *x, float radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_MSG("This interface is abandoned yet."); FAISS_THROW_MSG("This interface is abandoned yet.");
} }
@ -101,7 +101,7 @@ void IndexFlat::range_search(faiss::Index::idx_t n,
float radius, float radius,
std::vector<faiss::RangeSearchPartialResult*>& result, std::vector<faiss::RangeSearchPartialResult*>& result,
size_t buffer_size, size_t buffer_size,
const faiss::BitsetView& bitset) { const faiss::BitsetView bitset) {
switch (metric_type) { switch (metric_type) {
case METRIC_INNER_PRODUCT: case METRIC_INNER_PRODUCT:
@ -280,7 +280,7 @@ void IndexFlatL2BaseShift::search (
idx_t k, idx_t k,
float *distances, float *distances,
idx_t *labels, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (shift.size() == ntotal); FAISS_THROW_IF_NOT (shift.size() == ntotal);
@ -364,7 +364,7 @@ static void reorder_2_heaps (
void IndexRefineFlat::search ( void IndexRefineFlat::search (
idx_t n, const float *x, idx_t k, idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
idx_t k_base = idx_t (k * k_factor); idx_t k_base = idx_t (k * k_factor);
@ -458,7 +458,7 @@ void IndexFlat1D::search (
idx_t k, idx_t k,
float *distances, float *distances,
idx_t *labels, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT_MSG (perm.size() == ntotal, FAISS_THROW_IF_NOT_MSG (perm.size() == ntotal,
"Call update_permutation before search"); "Call update_permutation before search");

12
core/src/index/thirdparty/faiss/IndexFlat.h vendored
View File

@ -36,7 +36,7 @@ struct IndexFlat: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void assign ( void assign (
idx_t n, idx_t n,
@ -49,7 +49,7 @@ struct IndexFlat: Index {
const float* x, const float* x,
float radius, float radius,
RangeSearchResult* result, RangeSearchResult* result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void range_search( void range_search(
idx_t n, idx_t n,
@ -57,7 +57,7 @@ struct IndexFlat: Index {
float radius, float radius,
std::vector<RangeSearchPartialResult*> &result, std::vector<RangeSearchPartialResult*> &result,
size_t buffer_size, size_t buffer_size,
const BitsetView& bitset = nullptr); // const override const BitsetView bitset = nullptr); // const override
void reconstruct(idx_t key, float* recons) const override; void reconstruct(idx_t key, float* recons) const override;
@ -124,7 +124,7 @@ struct IndexFlatL2BaseShift: IndexFlatL2 {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
}; };
@ -160,7 +160,7 @@ struct IndexRefineFlat: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
~IndexRefineFlat() override; ~IndexRefineFlat() override;
}; };
@ -189,7 +189,7 @@ struct IndexFlat1D:IndexFlatL2 {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
}; };

4
core/src/index/thirdparty/faiss/IndexHNSW.cpp vendored
View File

@ -284,7 +284,7 @@ void IndexHNSW::train(idx_t n, const float* x)
} }
void IndexHNSW::search (idx_t n, const float *x, idx_t k, void IndexHNSW::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, const BitsetView& bitset) const float *distances, idx_t *labels, const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT_MSG(storage, FAISS_THROW_IF_NOT_MSG(storage,
@ -1013,7 +1013,7 @@ int search_from_candidates_2(const HNSW & hnsw,
} // namespace } // namespace
void IndexHNSW2Level::search (idx_t n, const float *x, idx_t k, void IndexHNSW2Level::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, const BitsetView& bitset) const float *distances, idx_t *labels, const BitsetView bitset) const
{ {
if (dynamic_cast<const Index2Layer*>(storage)) { if (dynamic_cast<const Index2Layer*>(storage)) {
IndexHNSW::search (n, x, k, distances, labels); IndexHNSW::search (n, x, k, distances, labels);

4
core/src/index/thirdparty/faiss/IndexHNSW.h vendored
View File

@ -92,7 +92,7 @@ struct IndexHNSW : Index {
/// entry point for search /// entry point for search
void search (idx_t n, const float *x, idx_t k, void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct(idx_t key, float* recons) const override; void reconstruct(idx_t key, float* recons) const override;
@ -164,7 +164,7 @@ struct IndexHNSW2Level : IndexHNSW {
/// entry point for search /// entry point for search
void search (idx_t n, const float *x, idx_t k, void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
}; };

22
core/src/index/thirdparty/faiss/IndexIVF.cpp vendored
View File

@ -318,7 +318,7 @@ void IndexIVF::set_direct_map_type (DirectMap::Type type)
void IndexIVF::search (idx_t n, const float *x, idx_t k, void IndexIVF::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]); std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]);
std::unique_ptr<float[]> coarse_dis(new float[n * nprobe]); std::unique_ptr<float[]> coarse_dis(new float[n * nprobe]);
@ -346,7 +346,7 @@ void IndexIVF::search (idx_t n, const float *x, idx_t k,
void IndexIVF::search_without_codes (idx_t n, const float *x, void IndexIVF::search_without_codes (idx_t n, const float *x,
const uint8_t *arranged_codes, std::vector<size_t> prefix_sum, const uint8_t *arranged_codes, std::vector<size_t> prefix_sum,
bool is_sq8, idx_t k, float *distances, idx_t *labels, bool is_sq8, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset) const BitsetView bitset)
{ {
std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]); std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe]);
@ -374,7 +374,7 @@ void IndexIVF::search_without_codes (idx_t n, const float *x,
} }
#if 0 #if 0
void IndexIVF::get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView& bitset) { void IndexIVF::get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView bitset) {
make_direct_map(true); make_direct_map(true);
/* only get vector by 1 id */ /* only get vector by 1 id */
@ -387,7 +387,7 @@ void IndexIVF::get_vector_by_id (idx_t n, const idx_t *xid, float *x, const Bits
} }
void IndexIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, void IndexIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset) { const BitsetView bitset) {
make_direct_map(true); make_direct_map(true);
auto x = new float[n * d]; auto x = new float[n * d];
@ -406,7 +406,7 @@ void IndexIVF::search_preassigned (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
long nprobe = params ? params->nprobe : this->nprobe; long nprobe = params ? params->nprobe : this->nprobe;
long max_codes = params ? params->max_codes : this->max_codes; long max_codes = params ? params->max_codes : this->max_codes;
@ -462,7 +462,7 @@ void IndexIVF::search_preassigned (idx_t n, const float *x, idx_t k,
// set porperly) and storing results in simi and idxi // set porperly) and storing results in simi and idxi
auto scan_one_list = [&] (idx_t key, float coarse_dis_i, auto scan_one_list = [&] (idx_t key, float coarse_dis_i,
float *simi, idx_t *idxi, float *simi, idx_t *idxi,
const BitsetView& bitset) { const BitsetView bitset) {
if (key < 0) { if (key < 0) {
// not enough centroids for multiprobe // not enough centroids for multiprobe
@ -612,7 +612,7 @@ void IndexIVF::search_preassigned_without_codes (idx_t n, const float *x,
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
const BitsetView& bitset) const BitsetView bitset)
{ {
long nprobe = params ? params->nprobe : this->nprobe; long nprobe = params ? params->nprobe : this->nprobe;
long max_codes = params ? params->max_codes : this->max_codes; long max_codes = params ? params->max_codes : this->max_codes;
@ -667,7 +667,7 @@ void IndexIVF::search_preassigned_without_codes (idx_t n, const float *x,
// single list scan using the current scanner (with query // single list scan using the current scanner (with query
// set porperly) and storing results in simi and idxi // set porperly) and storing results in simi and idxi
auto scan_one_list = [&] (idx_t key, float coarse_dis_i, const uint8_t *arranged_codes, auto scan_one_list = [&] (idx_t key, float coarse_dis_i, const uint8_t *arranged_codes,
float *simi, idx_t *idxi, const BitsetView& bitset) { float *simi, idx_t *idxi, const BitsetView bitset) {
if (key < 0) { if (key < 0) {
// not enough centroids for multiprobe // not enough centroids for multiprobe
@ -812,7 +812,7 @@ void IndexIVF::search_preassigned_without_codes (idx_t n, const float *x,
void IndexIVF::range_search (idx_t nx, const float *x, float radius, void IndexIVF::range_search (idx_t nx, const float *x, float radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
std::unique_ptr<idx_t[]> keys (new idx_t[nx * nprobe]); std::unique_ptr<idx_t[]> keys (new idx_t[nx * nprobe]);
std::unique_ptr<float []> coarse_dis (new float[nx * nprobe]); std::unique_ptr<float []> coarse_dis (new float[nx * nprobe]);
@ -834,7 +834,7 @@ void IndexIVF::range_search_preassigned (
idx_t nx, const float *x, float radius, idx_t nx, const float *x, float radius,
const idx_t *keys, const float *coarse_dis, const idx_t *keys, const float *coarse_dis,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
size_t nlistv = 0, ndis = 0; size_t nlistv = 0, ndis = 0;
@ -1250,7 +1250,7 @@ void InvertedListScanner::scan_codes_range (size_t ,
const idx_t *, const idx_t *,
float , float ,
RangeQueryResult &, RangeQueryResult &,
const BitsetView&) const const BitsetView) const
{ {
FAISS_THROW_MSG ("scan_codes_range not implemented"); FAISS_THROW_MSG ("scan_codes_range not implemented");
} }

20
core/src/index/thirdparty/faiss/IndexIVF.h vendored
View File

@ -208,7 +208,7 @@ struct IndexIVF: Index, Level1Quantizer {
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params=nullptr, const IVFSearchParameters *params=nullptr,
const BitsetView& bitset = nullptr const BitsetView bitset = nullptr
) const; ) const;
/** Similar to search_preassigned, but does not store codes **/ /** Similar to search_preassigned, but does not store codes **/
@ -221,36 +221,36 @@ struct IndexIVF: Index, Level1Quantizer {
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params = nullptr, const IVFSearchParameters *params = nullptr,
const BitsetView& bitset = nullptr); const BitsetView bitset = nullptr);
/** assign the vectors, then call search_preassign */ /** assign the vectors, then call search_preassign */
void search (idx_t n, const float *x, idx_t k, void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/** Similar to search, but does not store codes **/ /** Similar to search, but does not store codes **/
void search_without_codes (idx_t n, const float *x, void search_without_codes (idx_t n, const float *x,
const uint8_t *arranged_codes, std::vector<size_t> prefix_sum, const uint8_t *arranged_codes, std::vector<size_t> prefix_sum,
bool is_sq8, idx_t k, float *distances, idx_t *labels, bool is_sq8, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr); const BitsetView bitset = nullptr);
#if 0 #if 0
/** get raw vectors by ids */ /** get raw vectors by ids */
void get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView& bitset = nullptr) override; void get_vector_by_id (idx_t n, const idx_t *xid, float *x, const BitsetView bitset = nullptr) override;
void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) override; const BitsetView bitset = nullptr) override;
#endif #endif
void range_search (idx_t n, const float* x, float radius, void range_search (idx_t n, const float* x, float radius,
RangeSearchResult* result, RangeSearchResult* result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void range_search_preassigned(idx_t nx, const float *x, float radius, void range_search_preassigned(idx_t nx, const float *x, float radius,
const idx_t *keys, const float *coarse_dis, const idx_t *keys, const float *coarse_dis,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
/// get a scanner for this index (store_pairs means ignore labels) /// get a scanner for this index (store_pairs means ignore labels)
virtual InvertedListScanner *get_InvertedListScanner ( virtual InvertedListScanner *get_InvertedListScanner (
@ -413,7 +413,7 @@ struct InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float *distances, idx_t *labels, float *distances, idx_t *labels,
size_t k, size_t k,
const BitsetView& bitset = nullptr) const = 0; const BitsetView bitset = nullptr) const = 0;
/** scan a set of codes, compute distances to current query and /** scan a set of codes, compute distances to current query and
* update results if distances are below radius * update results if distances are below radius
@ -424,7 +424,7 @@ struct InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float radius, float radius,
RangeQueryResult &result, RangeQueryResult &result,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
virtual ~InvertedListScanner () {} virtual ~InvertedListScanner () {}

8
core/src/index/thirdparty/faiss/IndexIVFFlat.cpp vendored
View File

@ -184,7 +184,7 @@ struct IVFFlatScanner: InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float *simi, idx_t *idxi, float *simi, idx_t *idxi,
size_t k, size_t k,
const BitsetView& bitset) const override const BitsetView bitset) const override
{ {
const float *list_vecs = (const float*)codes; const float *list_vecs = (const float*)codes;
size_t nup = 0; size_t nup = 0;
@ -208,7 +208,7 @@ struct IVFFlatScanner: InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float radius, float radius,
RangeQueryResult & res, RangeQueryResult & res,
const BitsetView& bitset = nullptr) const override const BitsetView bitset = nullptr) const override
{ {
const float *list_vecs = (const float*)codes; const float *list_vecs = (const float*)codes;
for (size_t j = 0; j < list_size; j++) { for (size_t j = 0; j < list_size; j++) {
@ -354,7 +354,7 @@ void IndexIVFFlatDedup::search_preassigned (
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT_MSG ( FAISS_THROW_IF_NOT_MSG (
!store_pairs, "store_pairs not supported in IVFDedup"); !store_pairs, "store_pairs not supported in IVFDedup");
@ -483,7 +483,7 @@ void IndexIVFFlatDedup::range_search(
const float* , const float* ,
float , float ,
RangeSearchResult* , RangeSearchResult* ,
const BitsetView&) const const BitsetView) const
{ {
FAISS_THROW_MSG ("not implemented"); FAISS_THROW_MSG ("not implemented");
} }

4
core/src/index/thirdparty/faiss/IndexIVFFlat.h vendored
View File

@ -81,7 +81,7 @@ struct IndexIVFFlatDedup: IndexIVFFlat {
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params=nullptr, const IVFSearchParameters *params=nullptr,
const BitsetView& bitset = nullptr const BitsetView bitset = nullptr
) const override; ) const override;
size_t remove_ids(const IDSelector& sel) override; size_t remove_ids(const IDSelector& sel) override;
@ -92,7 +92,7 @@ struct IndexIVFFlatDedup: IndexIVFFlat {
const float* x, const float* x,
float radius, float radius,
RangeSearchResult* result, RangeSearchResult* result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/// not implemented /// not implemented
void update_vectors (int nv, const idx_t *idx, const float *v) override; void update_vectors (int nv, const idx_t *idx, const float *v) override;

18
core/src/index/thirdparty/faiss/IndexIVFPQ.cpp vendored
View File

@ -801,7 +801,7 @@ struct KnnSearchResults {
size_t nup; size_t nup;
inline void add (idx_t j, float dis, const BitsetView& bitset = nullptr) { inline void add (idx_t j, float dis, const BitsetView bitset = nullptr) {
if (C::cmp (heap_sim[0], dis)) { if (C::cmp (heap_sim[0], dis)) {
idx_t id = ids ? ids[j] : lo_build (key, j); idx_t id = ids ? ids[j] : lo_build (key, j);
if (!bitset.empty() && bitset.test((faiss::ConcurrentBitset::id_type_t)id)) if (!bitset.empty() && bitset.test((faiss::ConcurrentBitset::id_type_t)id))
@ -822,7 +822,7 @@ struct RangeSearchResults {
float radius; float radius;
RangeQueryResult & rres; RangeQueryResult & rres;
inline void add (idx_t j, float dis, const faiss::BitsetView& bitset = nullptr) { inline void add (idx_t j, float dis, const faiss::BitsetView bitset = nullptr) {
if (C::cmp (radius, dis)) { if (C::cmp (radius, dis)) {
idx_t id = ids ? ids[j] : lo_build (key, j); idx_t id = ids ? ids[j] : lo_build (key, j);
rres.add (dis, id); rres.add (dis, id);
@ -872,7 +872,7 @@ struct IVFPQScannerT: QueryTables {
template<class SearchResultType> template<class SearchResultType>
void scan_list_with_table (size_t ncode, const uint8_t *codes, void scan_list_with_table (size_t ncode, const uint8_t *codes,
SearchResultType & res, SearchResultType & res,
const BitsetView& bitset = nullptr) const const BitsetView bitset = nullptr) const
{ {
for (size_t j = 0; j < ncode; j++) { for (size_t j = 0; j < ncode; j++) {
PQDecoder decoder(codes, pq.nbits); PQDecoder decoder(codes, pq.nbits);
@ -895,7 +895,7 @@ struct IVFPQScannerT: QueryTables {
template<class SearchResultType> template<class SearchResultType>
void scan_list_with_pointer (size_t ncode, const uint8_t *codes, void scan_list_with_pointer (size_t ncode, const uint8_t *codes,
SearchResultType & res, SearchResultType & res,
const faiss::BitsetView& bitset = nullptr) const const faiss::BitsetView bitset = nullptr) const
{ {
for (size_t j = 0; j < ncode; j++) { for (size_t j = 0; j < ncode; j++) {
PQDecoder decoder(codes, pq.nbits); PQDecoder decoder(codes, pq.nbits);
@ -918,7 +918,7 @@ struct IVFPQScannerT: QueryTables {
template<class SearchResultType> template<class SearchResultType>
void scan_on_the_fly_dist (size_t ncode, const uint8_t *codes, void scan_on_the_fly_dist (size_t ncode, const uint8_t *codes,
SearchResultType &res, SearchResultType &res,
const faiss::BitsetView& bitset = nullptr) const const faiss::BitsetView bitset = nullptr) const
{ {
const float *dvec; const float *dvec;
float dis0 = 0; float dis0 = 0;
@ -958,7 +958,7 @@ struct IVFPQScannerT: QueryTables {
void scan_list_polysemous_hc ( void scan_list_polysemous_hc (
size_t ncode, const uint8_t *codes, size_t ncode, const uint8_t *codes,
SearchResultType & res, SearchResultType & res,
const faiss::BitsetView& bitset = nullptr) const const faiss::BitsetView bitset = nullptr) const
{ {
int ht = ivfpq.polysemous_ht; int ht = ivfpq.polysemous_ht;
size_t n_hamming_pass = 0, nup = 0; size_t n_hamming_pass = 0, nup = 0;
@ -996,7 +996,7 @@ struct IVFPQScannerT: QueryTables {
void scan_list_polysemous ( void scan_list_polysemous (
size_t ncode, const uint8_t *codes, size_t ncode, const uint8_t *codes,
SearchResultType &res, SearchResultType &res,
const faiss::BitsetView& bitset = nullptr) const const faiss::BitsetView bitset = nullptr) const
{ {
switch (pq.code_size) { switch (pq.code_size) {
#define HANDLE_CODE_SIZE(cs) \ #define HANDLE_CODE_SIZE(cs) \
@ -1070,7 +1070,7 @@ struct IVFPQScanner:
const idx_t *ids, const idx_t *ids,
float *heap_sim, idx_t *heap_ids, float *heap_sim, idx_t *heap_ids,
size_t k, size_t k,
const faiss::BitsetView& bitset) const override const faiss::BitsetView bitset) const override
{ {
KnnSearchResults<C> res = { KnnSearchResults<C> res = {
/* key */ this->key, /* key */ this->key,
@ -1101,7 +1101,7 @@ struct IVFPQScanner:
const idx_t *ids, const idx_t *ids,
float radius, float radius,
RangeQueryResult & rres, RangeQueryResult & rres,
const faiss::BitsetView& bitset = nullptr) const override const faiss::BitsetView bitset = nullptr) const override
{ {
RangeSearchResults<C> res = { RangeSearchResults<C> res = {
/* key */ this->key, /* key */ this->key,

2
core/src/index/thirdparty/faiss/IndexIVFPQR.cpp vendored
View File

@ -101,7 +101,7 @@ void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params, const IVFSearchParameters *params,
const BitsetView& bitset const BitsetView bitset
) const ) const
{ {
uint64_t t0; uint64_t t0;

2
core/src/index/thirdparty/faiss/IndexIVFPQR.h vendored
View File

@ -56,7 +56,7 @@ struct IndexIVFPQR: IndexIVFPQ {
float *distances, idx_t *labels, float *distances, idx_t *labels,
bool store_pairs, bool store_pairs,
const IVFSearchParameters *params=nullptr, const IVFSearchParameters *params=nullptr,
const BitsetView& bitset = nullptr const BitsetView bitset = nullptr
) const override; ) const override;
IndexIVFPQR(); IndexIVFPQR();

4
core/src/index/thirdparty/faiss/IndexIVFSpectralHash.cpp vendored
View File

@ -262,7 +262,7 @@ struct IVFScanner: InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float *simi, idx_t *idxi, float *simi, idx_t *idxi,
size_t k, size_t k,
const BitsetView& bitset) const override const BitsetView bitset) const override
{ {
size_t nup = 0; size_t nup = 0;
for (size_t j = 0; j < list_size; j++) { for (size_t j = 0; j < list_size; j++) {
@ -285,7 +285,7 @@ struct IVFScanner: InvertedListScanner {
const idx_t *ids, const idx_t *ids,
float radius, float radius,
RangeQueryResult & res, RangeQueryResult & res,
const BitsetView& bitset = nullptr) const override const BitsetView bitset = nullptr) const override
{ {
for (size_t j = 0; j < list_size; j++) { for (size_t j = 0; j < list_size; j++) {
float dis = hc.compute (codes); float dis = hc.compute (codes);

2
core/src/index/thirdparty/faiss/IndexLSH.cpp vendored
View File

@ -133,7 +133,7 @@ void IndexLSH::search (
idx_t k, idx_t k,
float *distances, float *distances,
idx_t *labels, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
const float *xt = apply_preprocess (n, x); const float *xt = apply_preprocess (n, x);

2
core/src/index/thirdparty/faiss/IndexLSH.h vendored
View File

@ -58,7 +58,7 @@ struct IndexLSH:Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reset() override; void reset() override;

2
core/src/index/thirdparty/faiss/IndexLattice.cpp vendored
View File

@ -128,7 +128,7 @@ void IndexLattice::add(idx_t , const float* )
void IndexLattice::search(idx_t , const float* , idx_t , void IndexLattice::search(idx_t , const float* , idx_t ,
float* , idx_t* , const BitsetView&) const float* , idx_t* , const BitsetView) const
{ {
FAISS_THROW_MSG("not implemented"); FAISS_THROW_MSG("not implemented");
} }

2
core/src/index/thirdparty/faiss/IndexLattice.h vendored
View File

@ -59,7 +59,7 @@ struct IndexLattice: Index {
void add(idx_t n, const float* x) override; void add(idx_t n, const float* x) override;
void search(idx_t n, const float* x, idx_t k, void search(idx_t n, const float* x, idx_t k,
float* distances, idx_t* labels, float* distances, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reset() override; void reset() override;
}; };

6
core/src/index/thirdparty/faiss/IndexPQ.cpp vendored
View File

@ -202,7 +202,7 @@ DistanceComputer * IndexPQ::get_distance_computer() const {
******************************************/ ******************************************/
void IndexPQ::search (idx_t n, const float *x, idx_t k, void IndexPQ::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
if (search_type == ST_PQ) { // Simple PQ search if (search_type == ST_PQ) { // Simple PQ search
@ -941,7 +941,7 @@ void MultiIndexQuantizer::train(idx_t n, const float *x)
void MultiIndexQuantizer::search (idx_t n, const float *x, idx_t k, void MultiIndexQuantizer::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
if (n == 0) return; if (n == 0) return;
// the allocation just below can be severe... // the allocation just below can be severe...
@ -1095,7 +1095,7 @@ void MultiIndexQuantizer2::train(idx_t n, const float* x)
void MultiIndexQuantizer2::search( void MultiIndexQuantizer2::search(
idx_t n, const float* x, idx_t K, idx_t n, const float* x, idx_t K,
float* distances, idx_t* labels, float* distances, idx_t* labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
if (n == 0) return; if (n == 0) return;

6
core/src/index/thirdparty/faiss/IndexPQ.h vendored
View File

@ -54,7 +54,7 @@ struct IndexPQ: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reset() override; void reset() override;
@ -159,7 +159,7 @@ struct MultiIndexQuantizer: Index {
void search( void search(
idx_t n, const float* x, idx_t k, idx_t n, const float* x, idx_t k,
float* distances, idx_t* labels, float* distances, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/// add and reset will crash at runtime /// add and reset will crash at runtime
void add(idx_t n, const float* x) override; void add(idx_t n, const float* x) override;
@ -193,7 +193,7 @@ struct MultiIndexQuantizer2: MultiIndexQuantizer {
void search( void search(
idx_t n, const float* x, idx_t k, idx_t n, const float* x, idx_t k,
float* distances, idx_t* labels, float* distances, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
}; };

4
core/src/index/thirdparty/faiss/IndexPreTransform.cpp vendored
View File

@ -181,7 +181,7 @@ void IndexPreTransform::add_with_ids (idx_t n, const float * x,
void IndexPreTransform::search (idx_t n, const float *x, idx_t k, void IndexPreTransform::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
const float *xt = apply_chain (n, x); const float *xt = apply_chain (n, x);
@ -191,7 +191,7 @@ void IndexPreTransform::search (idx_t n, const float *x, idx_t k,
void IndexPreTransform::range_search (idx_t n, const float* x, float radius, void IndexPreTransform::range_search (idx_t n, const float* x, float radius,
RangeSearchResult* result, RangeSearchResult* result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
const float *xt = apply_chain (n, x); const float *xt = apply_chain (n, x);

4
core/src/index/thirdparty/faiss/IndexPreTransform.h vendored
View File

@ -52,13 +52,13 @@ struct IndexPreTransform: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/* range search, no attempt is done to change the radius */ /* range search, no attempt is done to change the radius */
void range_search (idx_t n, const float* x, float radius, void range_search (idx_t n, const float* x, float radius,
RangeSearchResult* result, RangeSearchResult* result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct (idx_t key, float * recons) const override; void reconstruct (idx_t key, float * recons) const override;

4
core/src/index/thirdparty/faiss/IndexRHNSW.cpp vendored
View File

@ -237,7 +237,7 @@ void IndexRHNSW::train(idx_t n, const float* x)
} }
void IndexRHNSW::search (idx_t n, const float *x, idx_t k, void IndexRHNSW::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, const BitsetView& bitset) const float *distances, idx_t *labels, const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT_MSG(storage, FAISS_THROW_IF_NOT_MSG(storage,
@ -739,7 +739,7 @@ int search_from_candidates_2(const RHNSW & hnsw,
} // namespace } // namespace
void IndexRHNSW2Level::search (idx_t n, const float *x, idx_t k, void IndexRHNSW2Level::search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, const BitsetView& bitset) const float *distances, idx_t *labels, const BitsetView bitset) const
{ {
if (dynamic_cast<const Index2Layer*>(storage)) { if (dynamic_cast<const Index2Layer*>(storage)) {
IndexRHNSW::search (n, x, k, distances, labels); IndexRHNSW::search (n, x, k, distances, labels);

4
core/src/index/thirdparty/faiss/IndexRHNSW.h vendored
View File

@ -93,7 +93,7 @@ struct IndexRHNSW : Index {
/// entry point for search /// entry point for search
void search (idx_t n, const float *x, idx_t k, void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct(idx_t key, float* recons) const override; void reconstruct(idx_t key, float* recons) const override;
@ -151,7 +151,7 @@ struct IndexRHNSW2Level : IndexRHNSW {
/// entry point for search /// entry point for search
void search (idx_t n, const float *x, idx_t k, void search (idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
size_t cal_size(); size_t cal_size();
}; };

2
core/src/index/thirdparty/faiss/IndexReplicas.cpp vendored
View File

@ -81,7 +81,7 @@ IndexReplicasTemplate<IndexT>::search(idx_t n,
idx_t k, idx_t k,
distance_t* distances, distance_t* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
FAISS_THROW_IF_NOT_MSG(this->count() > 0, "no replicas in index"); FAISS_THROW_IF_NOT_MSG(this->count() > 0, "no replicas in index");
if (n == 0) { if (n == 0) {

2
core/src/index/thirdparty/faiss/IndexReplicas.h vendored
View File

@ -61,7 +61,7 @@ class IndexReplicasTemplate : public ThreadedIndex<IndexT> {
idx_t k, idx_t k,
distance_t* distances, distance_t* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/// reconstructs from the first index /// reconstructs from the first index
void reconstruct(idx_t, component_t *v) const override; void reconstruct(idx_t, component_t *v) const override;

2
core/src/index/thirdparty/faiss/IndexScalarQuantizer.cpp vendored
View File

@ -66,7 +66,7 @@ void IndexScalarQuantizer::search(
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT (is_trained); FAISS_THROW_IF_NOT (is_trained);
FAISS_THROW_IF_NOT (metric_type == METRIC_L2 || FAISS_THROW_IF_NOT (metric_type == METRIC_L2 ||

2
core/src/index/thirdparty/faiss/IndexScalarQuantizer.h vendored
View File

@ -59,7 +59,7 @@ struct IndexScalarQuantizer: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reset() override; void reset() override;

2
core/src/index/thirdparty/faiss/IndexShards.cpp vendored
View File

@ -265,7 +265,7 @@ IndexShardsTemplate<IndexT>::search(idx_t n,
idx_t k, idx_t k,
distance_t *distances, distance_t *distances,
idx_t *labels, idx_t *labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
long nshard = this->count(); long nshard = this->count();
std::vector<distance_t> all_distances(nshard * k * n); std::vector<distance_t> all_distances(nshard * k * n);

2
core/src/index/thirdparty/faiss/IndexShards.h vendored
View File

@ -76,7 +76,7 @@ struct IndexShardsTemplate : public ThreadedIndex<IndexT> {
void search(idx_t n, const component_t* x, idx_t k, void search(idx_t n, const component_t* x, idx_t k,
distance_t* distances, idx_t* labels, distance_t* distances, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void train(idx_t n, const component_t* x) override; void train(idx_t n, const component_t* x) override;

10
core/src/index/thirdparty/faiss/MetaIndexes.cpp vendored
View File

@ -83,7 +83,7 @@ template <typename IndexT>
void IndexIDMapTemplate<IndexT>::search void IndexIDMapTemplate<IndexT>::search
(idx_t n, const typename IndexT::component_t *x, idx_t k, (idx_t n, const typename IndexT::component_t *x, idx_t k,
typename IndexT::distance_t *distances, typename IndexT::idx_t *labels, typename IndexT::distance_t *distances, typename IndexT::idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
index->search (n, x, k, distances, labels, bitset); index->search (n, x, k, distances, labels, bitset);
idx_t *li = labels; idx_t *li = labels;
@ -96,7 +96,7 @@ void IndexIDMapTemplate<IndexT>::search
#if 0 #if 0
template <typename IndexT> template <typename IndexT>
void IndexIDMapTemplate<IndexT>::get_vector_by_id(idx_t n, const idx_t *xid, component_t *x, void IndexIDMapTemplate<IndexT>::get_vector_by_id(idx_t n, const idx_t *xid, component_t *x,
const BitsetView& bitset) const BitsetView bitset)
{ {
/* only get vector by 1 id */ /* only get vector by 1 id */
FAISS_ASSERT(n == 1); FAISS_ASSERT(n == 1);
@ -109,7 +109,7 @@ void IndexIDMapTemplate<IndexT>::get_vector_by_id(idx_t n, const idx_t *xid, com
template <typename IndexT> template <typename IndexT>
void IndexIDMapTemplate<IndexT>::search_by_id (idx_t n, const idx_t *xid, idx_t k, void IndexIDMapTemplate<IndexT>::search_by_id (idx_t n, const idx_t *xid, idx_t k,
typename IndexT::distance_t *distances, idx_t *labels, const BitsetView& bitset) typename IndexT::distance_t *distances, idx_t *labels, const BitsetView bitset)
{ {
auto x = new typename IndexT::component_t[n * IndexT::d]; auto x = new typename IndexT::component_t[n * IndexT::d];
for (idx_t i = 0; i < n; i++) { for (idx_t i = 0; i < n; i++) {
@ -124,7 +124,7 @@ template <typename IndexT>
void IndexIDMapTemplate<IndexT>::range_search void IndexIDMapTemplate<IndexT>::range_search
(typename IndexT::idx_t n, const typename IndexT::component_t *x, (typename IndexT::idx_t n, const typename IndexT::component_t *x,
typename IndexT::distance_t radius, RangeSearchResult *result, typename IndexT::distance_t radius, RangeSearchResult *result,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
index->range_search(n, x, radius, result, bitset); index->range_search(n, x, radius, result, bitset);
#pragma omp parallel for #pragma omp parallel for
@ -284,7 +284,7 @@ void IndexSplitVectors::add(idx_t /*n*/, const float* /*x*/) {
void IndexSplitVectors::search ( void IndexSplitVectors::search (
idx_t n, const float *x, idx_t k, idx_t n, const float *x, idx_t k,
float *distances, idx_t *labels, float *distances, idx_t *labels,
const BitsetView& bitset) const const BitsetView bitset) const
{ {
FAISS_THROW_IF_NOT_MSG (k == 1, FAISS_THROW_IF_NOT_MSG (k == 1,
"search implemented only for k=1"); "search implemented only for k=1");

10
core/src/index/thirdparty/faiss/MetaIndexes.h vendored
View File

@ -40,13 +40,13 @@ struct IndexIDMapTemplate : IndexT {
void search( void search(
idx_t n, const component_t* x, idx_t k, idx_t n, const component_t* x, idx_t k,
distance_t* distances, idx_t* labels, distance_t* distances, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
#if 0 #if 0
void get_vector_by_id(idx_t n, const idx_t *xid, component_t *x, const BitsetView& bitset = nullptr) override; void get_vector_by_id(idx_t n, const idx_t *xid, component_t *x, const BitsetView bitset = nullptr) override;
void search_by_id (idx_t n, const idx_t *xid, idx_t k, distance_t *distances, idx_t *labels, void search_by_id (idx_t n, const idx_t *xid, idx_t k, distance_t *distances, idx_t *labels,
const BitsetView& bitset = nullptr) override; const BitsetView bitset = nullptr) override;
#endif #endif
void train(idx_t n, const component_t* x) override; void train(idx_t n, const component_t* x) override;
@ -58,7 +58,7 @@ struct IndexIDMapTemplate : IndexT {
void range_search (idx_t n, const component_t *x, distance_t radius, void range_search (idx_t n, const component_t *x, distance_t radius,
RangeSearchResult *result, RangeSearchResult *result,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
~IndexIDMapTemplate () override; ~IndexIDMapTemplate () override;
IndexIDMapTemplate () {own_fields=false; index=nullptr; } IndexIDMapTemplate () {own_fields=false; index=nullptr; }
@ -119,7 +119,7 @@ struct IndexSplitVectors: Index {
idx_t k, idx_t k,
float* distances, float* distances,
idx_t* labels, idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void train(idx_t n, const float* x) override; void train(idx_t n, const float* x) override;

6
core/src/index/thirdparty/faiss/gpu/GpuIndex.cu vendored
View File

@ -199,7 +199,7 @@ GpuIndex::search(Index::idx_t n,
Index::idx_t k, Index::idx_t k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
FAISS_THROW_IF_NOT_MSG(this->is_trained, "Index not trained"); FAISS_THROW_IF_NOT_MSG(this->is_trained, "Index not trained");
// For now, only support <= max int results // For now, only support <= max int results
@ -276,7 +276,7 @@ GpuIndex::searchNonPaged_(int n,
int k, int k,
float* outDistancesData, float* outDistancesData,
Index::idx_t* outIndicesData, Index::idx_t* outIndicesData,
const BitsetView& bitset) const { const BitsetView bitset) const {
auto stream = resources_->getDefaultStream(device_); auto stream = resources_->getDefaultStream(device_);
// Make sure arguments are on the device we desire; use temporary // Make sure arguments are on the device we desire; use temporary
@ -296,7 +296,7 @@ GpuIndex::searchFromCpuPaged_(int n,
int k, int k,
float* outDistancesData, float* outDistancesData,
Index::idx_t* outIndicesData, Index::idx_t* outIndicesData,
const BitsetView& bitset) const { const BitsetView bitset) const {
Tensor<float, 2, true> outDistances(outDistancesData, {n, k}); Tensor<float, 2, true> outDistances(outDistancesData, {n, k});
Tensor<Index::idx_t, 2, true> outIndices(outIndicesData, {n, k}); Tensor<Index::idx_t, 2, true> outIndices(outIndicesData, {n, k});

8
core/src/index/thirdparty/faiss/gpu/GpuIndex.h vendored
View File

@ -73,7 +73,7 @@ class GpuIndex : public faiss::Index {
Index::idx_t k, Index::idx_t k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/// Overridden to force GPU indices to provide their own GPU-friendly /// Overridden to force GPU indices to provide their own GPU-friendly
/// implementation /// implementation
@ -112,7 +112,7 @@ class GpuIndex : public faiss::Index {
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset = nullptr) const = 0; const BitsetView bitset = nullptr) const = 0;
private: private:
/// Handles paged adds if the add set is too large, passes to /// Handles paged adds if the add set is too large, passes to
@ -132,7 +132,7 @@ private:
int k, int k,
float* outDistancesData, float* outDistancesData,
Index::idx_t* outIndicesData, Index::idx_t* outIndicesData,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
/// Calls searchImpl_ for a single page of GPU-resident data, /// Calls searchImpl_ for a single page of GPU-resident data,
/// handling paging of the data and copies from the CPU /// handling paging of the data and copies from the CPU
@ -141,7 +141,7 @@ private:
int k, int k,
float* outDistancesData, float* outDistancesData,
Index::idx_t* outIndicesData, Index::idx_t* outIndicesData,
const BitsetView& bitset = nullptr) const; const BitsetView bitset = nullptr) const;
protected: protected:
/// Manages streams, cuBLAS handles and scratch memory for devices /// Manages streams, cuBLAS handles and scratch memory for devices

2
core/src/index/thirdparty/faiss/gpu/GpuIndexBinaryFlat.cu vendored
View File

@ -146,7 +146,7 @@ GpuIndexBinaryFlat::search(faiss::IndexBinary::idx_t n,
faiss::IndexBinary::idx_t k, faiss::IndexBinary::idx_t k,
int32_t* distances, int32_t* distances,
faiss::IndexBinary::idx_t* labels, faiss::IndexBinary::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
if (n == 0) { if (n == 0) {
return; return;
} }

2
core/src/index/thirdparty/faiss/gpu/GpuIndexBinaryFlat.h vendored
View File

@ -55,7 +55,7 @@ class GpuIndexBinaryFlat : public IndexBinary {
faiss::IndexBinary::idx_t k, faiss::IndexBinary::idx_t k,
int32_t* distances, int32_t* distances,
faiss::IndexBinary::idx_t* labels, faiss::IndexBinary::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
void reconstruct(faiss::IndexBinary::idx_t key, void reconstruct(faiss::IndexBinary::idx_t key,
uint8_t* recons) const override; uint8_t* recons) const override;

2
core/src/index/thirdparty/faiss/gpu/GpuIndexFlat.cu vendored
View File

@ -196,7 +196,7 @@ GpuIndexFlat::searchImpl_(int n,
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
auto stream = resources_->getDefaultStream(device_); auto stream = resources_->getDefaultStream(device_);
// Input and output data are already resident on the GPU // Input and output data are already resident on the GPU

2
core/src/index/thirdparty/faiss/gpu/GpuIndexFlat.h vendored
View File

@ -122,7 +122,7 @@ class GpuIndexFlat : public GpuIndex {
int k, int k,
float* distances, float* distances,
faiss::Index::idx_t* labels, faiss::Index::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
protected: protected:
/// Our config object /// Our config object

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFFlat.cu vendored
View File

@ -300,7 +300,7 @@
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
// Device is already set in GpuIndex::search // Device is already set in GpuIndex::search
FAISS_ASSERT(index_); FAISS_ASSERT(index_);
FAISS_ASSERT(n > 0); FAISS_ASSERT(n > 0);

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFFlat.h vendored
View File

@ -76,7 +76,7 @@ class GpuIndexIVFFlat : public GpuIndexIVF {
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
private: private:
GpuIndexIVFFlatConfig ivfFlatConfig_; GpuIndexIVFFlatConfig ivfFlatConfig_;

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFPQ.cu vendored
View File

@ -359,7 +359,7 @@ GpuIndexIVFPQ::searchImpl_(int n,
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
// Device is already set in GpuIndex::search // Device is already set in GpuIndex::search
FAISS_ASSERT(index_); FAISS_ASSERT(index_);
FAISS_ASSERT(n > 0); FAISS_ASSERT(n > 0);

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFPQ.h vendored
View File

@ -118,7 +118,7 @@ class GpuIndexIVFPQ : public GpuIndexIVF {
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
private: private:
void verifySettings_() const; void verifySettings_() const;

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFSQHybrid.cu vendored
View File

@ -330,7 +330,7 @@ GpuIndexIVFSQHybrid::searchImpl_(int n,
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
// Device is already set in GpuIndex::search // Device is already set in GpuIndex::search
FAISS_ASSERT(index_); FAISS_ASSERT(index_);
FAISS_ASSERT(n > 0); FAISS_ASSERT(n > 0);

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFSQHybrid.h vendored
View File

@ -80,7 +80,7 @@ class GpuIndexIVFSQHybrid : public GpuIndexIVF {
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset = nullptr) const override; const BitsetView bitset = nullptr) const override;
/// Called from train to handle SQ residual training /// Called from train to handle SQ residual training
void trainResiduals_(Index::idx_t n, const float* x); void trainResiduals_(Index::idx_t n, const float* x);

2
core/src/index/thirdparty/faiss/gpu/GpuIndexIVFScalarQuantizer.cu vendored
View File

@ -342,7 +342,7 @@ GpuIndexIVFScalarQuantizer::searchImpl_(int n,
int k, int k,
float* distances, float* distances,
Index::idx_t* labels, Index::idx_t* labels,
const BitsetView& bitset) const { const BitsetView bitset) const {
// Device is already set in GpuIndex::search // Device is already set in GpuIndex::search
FAISS_ASSERT(index_); FAISS_ASSERT(index_);
FAISS_ASSERT(n > 0); FAISS_ASSERT(n > 0);

Some files were not shown because too many files have changed in this diff Show More